{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read Data Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:34:33.155674Z",
     "start_time": "2017-07-23T21:34:32.762800Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from collections import namedtuple\n",
    "pd.set_option(\"display.max_rows\",1000)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:34:33.166129Z",
     "start_time": "2017-07-23T21:34:33.157119Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "rm: cannot remove 'dataset/tf_vae_dense_trained_together_nsl_kdd_all-.pkl': No such file or directory\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "rm dataset/scores/tf_vae_dense_trained_together_nsl_kdd_all-.pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:34:33.612202Z",
     "start_time": "2017-07-23T21:34:33.167664Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class dataset:\n",
    "    kdd_train_2labels = pd.read_pickle(\"dataset/kdd_train__2labels.pkl\")\n",
    "    kdd_test_2labels = pd.read_pickle(\"dataset/kdd_test_2labels.pkl\")\n",
    "    kdd_test__2labels = pd.read_pickle(\"dataset/kdd_test__2labels.pkl\")\n",
    "    \n",
    "    kdd_train_5labels = pd.read_pickle(\"dataset/kdd_train_5labels.pkl\")\n",
    "    kdd_test_5labels = pd.read_pickle(\"dataset/kdd_test_5labels.pkl\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:34:33.619100Z",
     "start_time": "2017-07-23T21:34:33.613759Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25192, 124)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_train_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:34:33.625379Z",
     "start_time": "2017-07-23T21:34:33.620561Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22544, 124)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_test_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:34:34.024033Z",
     "start_time": "2017-07-23T21:34:33.626772Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97509982675167528"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import model_selection as ms\n",
    "from sklearn import preprocessing as pp\n",
    "\n",
    "class preprocess:\n",
    "    \n",
    "    output_columns_2labels = ['is_Normal','is_Attack']\n",
    "    \n",
    "    x_input = dataset.kdd_train_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_output = dataset.kdd_train_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    x_test_input = dataset.kdd_test_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test = dataset.kdd_test_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    x_test__input = dataset.kdd_test__2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test_ = dataset.kdd_test__2labels.loc[:,output_columns_2labels]\n",
    "    \n",
    "    ss = pp.StandardScaler()\n",
    "\n",
    "    x_train = ss.fit_transform(x_input)\n",
    "    x_test = ss.transform(x_test_input)\n",
    "    x_test_ = ss.transform(x_test__input)\n",
    "\n",
    "    y_train = y_output.values\n",
    "    y_test = y_test.values\n",
    "    y_test_ = y_test_.values\n",
    "\n",
    "preprocess.x_train.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:34:35.146610Z",
     "start_time": "2017-07-23T21:34:34.025620Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:34:35.623484Z",
     "start_time": "2017-07-23T21:34:35.148163Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class network(object):\n",
    "    \n",
    "    input_dim = 122\n",
    "    classes = 2\n",
    "    hidden_encoder_dim = 122\n",
    "    hidden_layers = 1\n",
    "    latent_dim = 10\n",
    "\n",
    "    hidden_decoder_dim = 122\n",
    "    lam = 0.001\n",
    "    \n",
    "    def __init__(self, classes, hidden_layers, num_of_features):\n",
    "        self.classes = classes\n",
    "        self.hidden_layers = hidden_layers\n",
    "        self.latent_dim = num_of_features\n",
    "            \n",
    "    def build_layers(self):\n",
    "        tf.reset_default_graph()\n",
    "        #learning_rate = tf.Variable(initial_value=0.001)\n",
    "\n",
    "        input_dim = self.input_dim\n",
    "        classes = self.classes\n",
    "        hidden_encoder_dim = self.hidden_encoder_dim\n",
    "        hidden_layers = self.hidden_layers\n",
    "        latent_dim = self.latent_dim\n",
    "        hidden_decoder_dim = self.hidden_decoder_dim\n",
    "        lam = self.lam\n",
    "        \n",
    "        with tf.variable_scope(\"Input\"):\n",
    "            self.x = tf.placeholder(\"float\", shape=[None, input_dim])\n",
    "            self.y_ = tf.placeholder(\"float\", shape=[None, classes])\n",
    "            self.keep_prob = tf.placeholder(\"float\")\n",
    "            self.lr = tf.placeholder(\"float\")\n",
    "        \n",
    "        with tf.variable_scope(\"Layer_Encoder\"):\n",
    "\n",
    "            hidden_encoder = tf.layers.dense(self.x, hidden_encoder_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)\n",
    "            hidden_encoder = tf.nn.dropout(hidden_encoder, self.keep_prob)\n",
    "            for h in range(hidden_layers - 1):\n",
    "                hidden_encoder = tf.layers.dense(hidden_encoder, hidden_encoder_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)\n",
    "                hidden_encoder = tf.nn.dropout(hidden_encoder, self.keep_prob)\n",
    "                \n",
    "        with tf.variable_scope(\"Layer_Mean\"):\n",
    "            mu_encoder = tf.layers.dense(hidden_encoder, latent_dim, activation = None, kernel_regularizer=tf.nn.l2_loss)\n",
    "\n",
    "        with tf.variable_scope(\"Layer_Variance\"):\n",
    "            logvar_encoder = tf.layers.dense(hidden_encoder, latent_dim, activation = None, kernel_regularizer=tf.nn.l2_loss)\n",
    "\n",
    "        with tf.variable_scope(\"Sampling_Distribution\"):\n",
    "            # Sample epsilon\n",
    "            epsilon = tf.random_normal(tf.shape(logvar_encoder), mean=0, stddev=1, name='epsilon')\n",
    "\n",
    "            # Sample latent variable\n",
    "            std_encoder = tf.exp(0.5 * logvar_encoder)\n",
    "            z = mu_encoder + tf.multiply(std_encoder, epsilon)\n",
    "            \n",
    "            #tf.summary.histogram(\"Sample_Distribution\", z)\n",
    "\n",
    "        with tf.variable_scope(\"Layer_Decoder\"):\n",
    "            hidden_decoder = tf.layers.dense(z, hidden_decoder_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)\n",
    "            hidden_decoder = tf.nn.dropout(hidden_decoder, self.keep_prob)\n",
    "            for h in range(hidden_layers - 1):\n",
    "                hidden_decoder = tf.layers.dense(hidden_decoder, hidden_decoder_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)\n",
    "                hidden_decoder = tf.nn.dropout(hidden_decoder, self.keep_prob)\n",
    "                \n",
    "        with tf.variable_scope(\"Layer_Reconstruction\"):\n",
    "            x_hat = tf.layers.dense(hidden_decoder, input_dim, activation = None)\n",
    "            \n",
    "        with tf.variable_scope(\"Layer_Dense_Hidden\"):\n",
    "            hidden_output = tf.layers.dense(z,latent_dim, activation=tf.nn.relu)\n",
    "\n",
    "        with tf.variable_scope(\"Layer_Dense_Softmax\"):\n",
    "            self.y = tf.layers.dense(z, classes, activation=tf.nn.softmax)\n",
    "\n",
    "        with tf.variable_scope(\"Loss\"):\n",
    "            \n",
    "            BCE = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=x_hat, labels=self.x), reduction_indices=1)\n",
    "            KLD = -0.5 * tf.reduce_mean(1 + logvar_encoder - tf.pow(mu_encoder, 2) - tf.exp(logvar_encoder), reduction_indices=1)\n",
    "            softmax_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = self.y_, logits = self.y))\n",
    "\n",
    "            loss = tf.reduce_mean((BCE + KLD + softmax_loss) * lam)\n",
    "\n",
    "            loss = tf.clip_by_value(loss, -1e-4, 1e-4)\n",
    "            loss = tf.where(tf.is_nan(loss), 1e-4, loss)\n",
    "            loss = tf.where(tf.equal(loss, -1e-4), tf.random_normal(loss.shape), loss)\n",
    "            loss = tf.where(tf.equal(loss, 1e-4), tf.random_normal(loss.shape), loss)\n",
    "            \n",
    "            self.regularized_loss = tf.abs(loss, name = \"Regularized_loss\")\n",
    "            correct_prediction = tf.equal(tf.argmax(self.y_, 1), tf.argmax(self.y, 1))\n",
    "            self.tf_accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name = \"Accuracy\")\n",
    "\n",
    "        with tf.variable_scope(\"Optimizer\"):\n",
    "            learning_rate=self.lr #1e-2\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            gradients, variables = zip(*optimizer.compute_gradients(self.regularized_loss))\n",
    "            gradients = [\n",
    "                None if gradient is None else tf.clip_by_value(gradient, -1, 1)\n",
    "                for gradient in gradients]\n",
    "            self.train_op = optimizer.apply_gradients(zip(gradients, variables))\n",
    "            #self.train_op = optimizer.minimize(self.regularized_loss)\n",
    "            \n",
    "        # add op for merging summary\n",
    "        #self.summary_op = tf.summary.merge_all()\n",
    "        self.pred = tf.argmax(self.y, axis = 1)\n",
    "        self.actual = tf.argmax(self.y_, axis = 1)\n",
    "\n",
    "        # add Saver ops\n",
    "        self.saver = tf.train.Saver()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:34:35.966440Z",
     "start_time": "2017-07-23T21:34:35.625232Z"
    },
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "import time\n",
    "import sklearn.metrics as me \n",
    "\n",
    "class Train:    \n",
    "    \n",
    "    result = namedtuple(\"score\", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score', 'f1_score', 'test_score_20', 'f1_score_20', 'time_taken'])\n",
    "\n",
    "    predictions = {}\n",
    "    predictions_ = {}\n",
    "\n",
    "    results = []\n",
    "    best_acc = 0\n",
    "    best_acc_global = 0\n",
    "\n",
    "    def train(epochs, net, h,f, lrs):\n",
    "        batch_iterations = 200\n",
    "        train_loss = None\n",
    "        Train.best_acc = 0\n",
    "        os.makedirs(\"dataset/tf_vae_dense_trained_together_nsl_kdd-/hidden layers_{}_features count_{}\".format(h,f),\n",
    "                    exist_ok = True)\n",
    "\n",
    "        with tf.Session() as sess:\n",
    "            #summary_writer_train = tf.summary.FileWriter('./logs/kdd/VAE/training', graph=sess.graph)\n",
    "            #summary_writer_valid = tf.summary.FileWriter('./logs/kdd/VAE/validation')\n",
    "\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            start_time = time.perf_counter()\n",
    "\n",
    "            Train.best_acc = 0\n",
    "            for c, lr in enumerate(lrs):\n",
    "                for epoch in range(1, (epochs+1)):\n",
    "                    x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, \n",
    "                                                                              preprocess.y_train, \n",
    "                                                                              test_size=0.2)\n",
    "                    batch_indices = np.array_split(np.arange(x_train.shape[0]), \n",
    "                                               batch_iterations)\n",
    "\n",
    "                    for i in batch_indices:\n",
    "\n",
    "                        def train_batch():\n",
    "                            nonlocal train_loss\n",
    "                            _, train_loss = sess.run([net.train_op, \n",
    "                                                                   net.regularized_loss, \n",
    "                                                                   ], #net.summary_op\n",
    "                                                                  feed_dict={net.x: x_train[i,:], \n",
    "                                                                             net.y_: y_train[i,:], \n",
    "                                                                             net.keep_prob:1, net.lr:lr})\n",
    "\n",
    "                        train_batch()\n",
    "                        count = 10\n",
    "                        \n",
    "                        while((train_loss > 1e4 or np.isnan(train_loss)) and epoch > 1 and count > 1):\n",
    "                            print(\"Step {} | High Training Loss: {:.6f} ... Restoring Net\".format(epoch, train_loss))\n",
    "                            net.saver.restore(sess, \n",
    "                                              tf.train.latest_checkpoint('dataset/tf_vae_dense_trained_together_nsl_kdd-/hidden layers_{}_features count_{}'\n",
    "                                                                         .format(h,f)))\n",
    "                            train_batch()\n",
    "                            count -= 1\n",
    "\n",
    "                    valid_loss, valid_accuracy = sess.run([net.regularized_loss, net.tf_accuracy], #net.summary_op\n",
    "                                                              feed_dict={net.x: x_valid, \n",
    "                                                                         net.y_: y_valid, \n",
    "                                                                         net.keep_prob:1, net.lr:lr})\n",
    "                    \n",
    "                    test_accuracy, test_loss, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, net.regularized_loss, net.pred, \n",
    "                                                                                      net.actual, net.y], #net.summary_op \n",
    "                                                                                      feed_dict={net.x: preprocess.x_test, \n",
    "                                                                                     net.y_: preprocess.y_test, \n",
    "                                                                                     net.keep_prob:1, net.lr:lr})\n",
    "                    \n",
    "                    f1_score = me.f1_score(actual_value, pred_value)\n",
    "                    test_accuracy_, test_loss_, pred_value_, actual_value_, y_pred_ = sess.run([net.tf_accuracy, net.regularized_loss, net.pred, \n",
    "                                                                                      net.actual, net.y], #net.summary_op \n",
    "                                                                                      feed_dict={net.x: preprocess.x_test_, \n",
    "                                                                                     net.y_: preprocess.y_test_, \n",
    "                                                                                     net.keep_prob:1, net.lr:lr})\n",
    "                    f1_score_ = me.f1_score(actual_value_, pred_value_)\n",
    "                    #summary_writer_valid.add_summary(summary_str, epoch)\n",
    "\n",
    "                    print(\"Step {} | Training Loss: {:.6f} | Validation Accuracy: {:.6f}\".format(epoch, train_loss, valid_accuracy))\n",
    "                    print(\"Accuracy on Test data: {}, {}\".format(test_accuracy, test_accuracy_))\n",
    "\n",
    "                    if test_accuracy > Train.best_acc_global:\n",
    "                        Train.best_acc_global = test_accuracy\n",
    "                        Train.pred_value = pred_value\n",
    "                        Train.actual_value = actual_value\n",
    "                        \n",
    "                        Train.pred_value_ = pred_value_\n",
    "                        Train.actual_value_ = actual_value_\n",
    "                        \n",
    "                        Train.best_parameters = \"Hidden Layers:{}, Features Count:{}\".format(h, f)\n",
    "\n",
    "                    if test_accuracy > Train.best_acc:\n",
    "                        Train.best_acc = test_accuracy\n",
    "\n",
    "                        if not (np.isnan(train_loss)):\n",
    "                            net.saver.save(sess, \n",
    "                                       \"dataset/tf_vae_dense_trained_together_nsl_kdd-/hidden layers_{}_features count_{}/model\"\n",
    "                                       .format(h,f), \n",
    "                                       global_step = epoch, \n",
    "                                       write_meta_graph=False)\n",
    "\n",
    "                        curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,-2], \"Normal_prob\":y_pred[:, -1], \"Prediction\":pred_value, \"Actual\":actual_value})\n",
    "                        curr_pred_ = pd.DataFrame({\"Attack_prob\":y_pred_[:,-2], \"Normal_prob\":y_pred_[:, -1], \"Prediction\":pred_value_, \"Actual\": actual_value_})\n",
    "                        \n",
    "                        Train.predictions.update({\"{}_{}_{}\".format((epoch+1)*(c+1),f,h):(curr_pred, \n",
    "                                                   Train.result((epoch+1)*(c+1), f, h, valid_accuracy, test_accuracy, f1_score, test_accuracy_, f1_score_, time.perf_counter() - start_time))})\n",
    "                        Train.predictions_.update({\"{}_{}_{}\".format((epoch+1)*(c+1),f,h):(curr_pred_, \n",
    "                                                   Train.result((epoch+1)*(c+1), f, h, valid_accuracy, test_accuracy, f1_score, test_accuracy_, f1_score_, time.perf_counter() - start_time))})\n",
    "                        #Train.results.append(Train.result(epochs, f, h,valid_accuracy, test_accuracy))\n",
    "            print(\"Best Accuracy on Test data: {}\".format(Train.best_acc))\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:34:36.056924Z",
     "start_time": "2017-07-23T21:34:35.968005Z"
    },
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "\n",
    "df_results = []\n",
    "past_scores = []\n",
    "\n",
    "class Hyperparameters:\n",
    "#    features_arr = [2, 4, 8, 16, 32, 64, 128, 256]\n",
    "#    hidden_layers_arr = [2, 4, 6, 10]\n",
    "\n",
    "    def start_training():\n",
    "        global df_results\n",
    "        global past_scores\n",
    "        \n",
    "        features_arr = [1, 12, 24, 48, 122]\n",
    "        hidden_layers_arr = [1, 3]\n",
    "\n",
    "        Train.predictions = {}\n",
    "        Train.predictions_ = {}\n",
    "        Train.results = []\n",
    "\n",
    "        epochs = [15]\n",
    "        lrs = [1e-2, 1e-2]\n",
    "\n",
    "        for e, h, f in itertools.product(epochs, hidden_layers_arr, features_arr):\n",
    "            print(\"Current Layer Attributes - epochs:{} hidden layers:{} features count:{}\".format(e,h,f))\n",
    "            n = network(2,h,f)\n",
    "            n.build_layers()\n",
    "            Train.train(e, n, h,f, lrs)\n",
    "            \n",
    "        dict1 = {}\n",
    "        dict1_ = {}\n",
    "        dict2 = []\n",
    "\n",
    "        for k, (v1, v2) in Train.predictions.items():\n",
    "            dict1.update({k: v1})\n",
    "            dict2.append(v2)\n",
    "\n",
    "        for k, (v1_, v2) in Train.predictions_.items():\n",
    "            dict1_.update({k: v1_})\n",
    "\n",
    "        Train.predictions = dict1\n",
    "        Train.predictions_ = dict1_\n",
    "        \n",
    "        Train.results = dict2\n",
    "        df_results = pd.DataFrame(Train.results)\n",
    "\n",
    "        #temp = df_results.set_index(['no_of_features', 'hidden_layers'])\n",
    "\n",
    "        \n",
    "        if not os.path.isfile('dataset/scores/tf_vae_dense_trained_together_nsl_kdd_all-.pkl'):\n",
    "            past_scores = df_results\n",
    "        else:\n",
    "            past_scores = pd.read_pickle(\"dataset/scores/tf_vae_dense_trained_together_nsl_kdd_all-.pkl\")\n",
    "            past_scores = past_scores.append(df_results, ignore_index=True)\n",
    "                \n",
    "        past_scores.to_pickle(\"dataset/scores/tf_vae_dense_trained_together_nsl_kdd_all-.pkl\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:41:04.335932Z",
     "start_time": "2017-07-23T21:34:36.058395Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Layer Attributes - epochs:15 hidden layers:1 features count:1\n",
      "Step 1 | Training Loss: 0.000665 | Validation Accuracy: 0.652113\n",
      "Accuracy on Test data: 0.6522799730300903, 0.6925738453865051\n",
      "Step 2 | Training Loss: 0.000305 | Validation Accuracy: 0.815836\n",
      "Accuracy on Test data: 0.716864824295044, 0.48877638578414917\n",
      "Step 3 | Training Loss: 0.000190 | Validation Accuracy: 0.802540\n",
      "Accuracy on Test data: 0.7153566479682922, 0.5171307921409607\n",
      "Step 4 | Training Loss: 0.000305 | Validation Accuracy: 0.773963\n",
      "Accuracy on Test data: 0.7897001504898071, 0.7081856727600098\n",
      "Step 5 | Training Loss: 0.000015 | Validation Accuracy: 0.752133\n",
      "Accuracy on Test data: 0.7259137630462646, 0.6094514727592468\n",
      "Step 6 | Training Loss: 0.000042 | Validation Accuracy: 0.818218\n",
      "Accuracy on Test data: 0.7488910555839539, 0.598143458366394\n",
      "Step 7 | Training Loss: 0.000106 | Validation Accuracy: 0.742012\n",
      "Accuracy on Test data: 0.6262863874435425, 0.45603376626968384\n",
      "Step 8 | Training Loss: 0.000142 | Validation Accuracy: 0.820004\n",
      "Accuracy on Test data: 0.6794712543487549, 0.4881856441497803\n",
      "Step 9 | Training Loss: 0.000031 | Validation Accuracy: 0.876563\n",
      "Accuracy on Test data: 0.7201915979385376, 0.4871729910373688\n",
      "Step 10 | Training Loss: 0.000224 | Validation Accuracy: 0.867236\n",
      "Accuracy on Test data: 0.7204577922821045, 0.5154430270195007\n",
      "Step 11 | Training Loss: 0.000171 | Validation Accuracy: 0.852947\n",
      "Accuracy on Test data: 0.7219215631484985, 0.5103797316551208\n",
      "Step 12 | Training Loss: 0.000192 | Validation Accuracy: 0.803136\n",
      "Accuracy on Test data: 0.7052430510520935, 0.5766244530677795\n",
      "Step 13 | Training Loss: 0.000263 | Validation Accuracy: 0.811074\n",
      "Accuracy on Test data: 0.6953513026237488, 0.5509704351425171\n",
      "Step 14 | Training Loss: 0.000170 | Validation Accuracy: 0.793411\n",
      "Accuracy on Test data: 0.6611958742141724, 0.46725738048553467\n",
      "Step 15 | Training Loss: 0.000350 | Validation Accuracy: 0.838857\n",
      "Accuracy on Test data: 0.7302608489990234, 0.5295358896255493\n",
      "Step 1 | Training Loss: 0.000066 | Validation Accuracy: 0.834888\n",
      "Accuracy on Test data: 0.7156227827072144, 0.5097046494483948\n",
      "Step 2 | Training Loss: 0.000019 | Validation Accuracy: 0.855725\n",
      "Accuracy on Test data: 0.7503548860549927, 0.5929113626480103\n",
      "Step 3 | Training Loss: 0.000022 | Validation Accuracy: 0.856718\n",
      "Accuracy on Test data: 0.778788149356842, 0.6087763905525208\n",
      "Step 4 | Training Loss: 0.000024 | Validation Accuracy: 0.845207\n",
      "Accuracy on Test data: 0.7721788287162781, 0.6077637076377869\n",
      "Step 5 | Training Loss: 0.000170 | Validation Accuracy: 0.857908\n",
      "Accuracy on Test data: 0.7426809668540955, 0.5760337710380554\n",
      "Step 6 | Training Loss: 0.000027 | Validation Accuracy: 0.800357\n",
      "Accuracy on Test data: 0.7191270589828491, 0.5897046327590942\n",
      "Step 7 | Training Loss: 0.000359 | Validation Accuracy: 0.854336\n",
      "Accuracy on Test data: 0.7472054362297058, 0.5720674991607666\n",
      "Step 8 | Training Loss: 0.000118 | Validation Accuracy: 0.871800\n",
      "Accuracy on Test data: 0.7754169702529907, 0.6183122396469116\n",
      "Step 9 | Training Loss: 0.000018 | Validation Accuracy: 0.873983\n",
      "Accuracy on Test data: 0.7435681223869324, 0.5558649897575378\n",
      "Step 10 | Training Loss: 0.000125 | Validation Accuracy: 0.857908\n",
      "Accuracy on Test data: 0.7341199517250061, 0.5496202707290649\n",
      "Step 11 | Training Loss: 0.000137 | Validation Accuracy: 0.859893\n",
      "Accuracy on Test data: 0.7411284446716309, 0.5871729850769043\n",
      "Step 12 | Training Loss: 0.000012 | Validation Accuracy: 0.868823\n",
      "Accuracy on Test data: 0.76987224817276, 0.6269198060035706\n",
      "Step 13 | Training Loss: 0.000013 | Validation Accuracy: 0.865846\n",
      "Accuracy on Test data: 0.7399308085441589, 0.5775527358055115\n",
      "Step 14 | Training Loss: 0.000078 | Validation Accuracy: 0.885295\n",
      "Accuracy on Test data: 0.7483587861061096, 0.5668354630470276\n",
      "Step 15 | Training Loss: 0.000042 | Validation Accuracy: 0.874975\n",
      "Accuracy on Test data: 0.7628637552261353, 0.6121519207954407\n",
      "Best Accuracy on Test data: 0.7897001504898071\n",
      "Current Layer Attributes - epochs:15 hidden layers:1 features count:12\n",
      "Step 1 | Training Loss: 1.987724 | Validation Accuracy: 0.815241\n",
      "Accuracy on Test data: 0.7488023638725281, 0.5691139101982117\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ritesh_malaiya/anaconda3/envs/p3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1113: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2 | Training Loss: 0.012218 | Validation Accuracy: 0.542568\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 3 | Training Loss: 1.805758 | Validation Accuracy: 0.538599\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 4 | Training Loss: 1.564975 | Validation Accuracy: 0.541774\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 0.934638 | Validation Accuracy: 0.525700\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 6 | Training Loss: 0.153852 | Validation Accuracy: 0.528279\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 7 | Training Loss: 0.682114 | Validation Accuracy: 0.544751\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 8 | Training Loss: 0.747098 | Validation Accuracy: 0.530859\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 9 | Training Loss: 0.797108 | Validation Accuracy: 0.534431\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 10 | Training Loss: 1.495139 | Validation Accuracy: 0.524707\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 11 | Training Loss: 0.579927 | Validation Accuracy: 0.527287\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 12 | Training Loss: 0.396832 | Validation Accuracy: 0.539591\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 13 | Training Loss: 0.592717 | Validation Accuracy: 0.534431\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 14 | Training Loss: 0.511182 | Validation Accuracy: 0.528875\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 15 | Training Loss: 0.804181 | Validation Accuracy: 0.529073\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 1 | Training Loss: 0.318750 | Validation Accuracy: 0.530462\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 2 | Training Loss: 0.295224 | Validation Accuracy: 0.536416\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 3 | Training Loss: 1.602901 | Validation Accuracy: 0.543362\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 4 | Training Loss: 0.663169 | Validation Accuracy: 0.534630\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 0.338673 | Validation Accuracy: 0.532248\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 6 | Training Loss: 1.041948 | Validation Accuracy: 0.540980\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 7 | Training Loss: 0.936222 | Validation Accuracy: 0.543759\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 8 | Training Loss: 0.630083 | Validation Accuracy: 0.528676\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 9 | Training Loss: 0.986195 | Validation Accuracy: 0.531852\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 10 | Training Loss: 0.356152 | Validation Accuracy: 0.537011\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 11 | Training Loss: 0.058322 | Validation Accuracy: 0.533836\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 12 | Training Loss: 0.735073 | Validation Accuracy: 0.531653\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 13 | Training Loss: 0.321524 | Validation Accuracy: 0.533836\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 14 | Training Loss: 0.654641 | Validation Accuracy: 0.527287\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 15 | Training Loss: 2.348252 | Validation Accuracy: 0.531653\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Best Accuracy on Test data: 0.7488023638725281\n",
      "Current Layer Attributes - epochs:15 hidden layers:1 features count:24\n",
      "Step 1 | Training Loss: 0.001060 | Validation Accuracy: 0.871998\n",
      "Accuracy on Test data: 0.6661639213562012, 0.45257383584976196\n",
      "Step 2 | Training Loss: 0.000838 | Validation Accuracy: 0.864854\n",
      "Accuracy on Test data: 0.7920954823493958, 0.6361181139945984\n",
      "Step 3 | Training Loss: 0.001454 | Validation Accuracy: 0.791030\n",
      "Accuracy on Test data: 0.6505500078201294, 0.3876793384552002\n",
      "Step 4 | Training Loss: 0.955112 | Validation Accuracy: 0.541774\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 0.983788 | Validation Accuracy: 0.529272\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 6 | Training Loss: 1.135413 | Validation Accuracy: 0.537408\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 7 | Training Loss: 0.508043 | Validation Accuracy: 0.535027\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 8 | Training Loss: 1.153921 | Validation Accuracy: 0.536019\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 9 | Training Loss: 0.184121 | Validation Accuracy: 0.528478\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 10 | Training Loss: 0.921163 | Validation Accuracy: 0.546537\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 11 | Training Loss: 2.288743 | Validation Accuracy: 0.533638\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 12 | Training Loss: 0.920667 | Validation Accuracy: 0.530264\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 13 | Training Loss: 0.536525 | Validation Accuracy: 0.538996\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 14 | Training Loss: 0.539061 | Validation Accuracy: 0.526692\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 15 | Training Loss: 0.954227 | Validation Accuracy: 0.535225\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 1 | Training Loss: 0.161555 | Validation Accuracy: 0.531455\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 2 | Training Loss: 0.406868 | Validation Accuracy: 0.530859\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 3 | Training Loss: 0.634016 | Validation Accuracy: 0.527684\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 4 | Training Loss: 1.316543 | Validation Accuracy: 0.533241\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 1.074220 | Validation Accuracy: 0.538400\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 6 | Training Loss: 0.439035 | Validation Accuracy: 0.524707\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 7 | Training Loss: 1.131871 | Validation Accuracy: 0.529272\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 8 | Training Loss: 0.295083 | Validation Accuracy: 0.539988\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 9 | Training Loss: 0.265419 | Validation Accuracy: 0.525303\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 10 | Training Loss: 0.232637 | Validation Accuracy: 0.546140\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 11 | Training Loss: 0.895969 | Validation Accuracy: 0.533836\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 12 | Training Loss: 1.249705 | Validation Accuracy: 0.526295\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 13 | Training Loss: 2.190820 | Validation Accuracy: 0.534035\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 14 | Training Loss: 0.695601 | Validation Accuracy: 0.529867\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 15 | Training Loss: 1.205935 | Validation Accuracy: 0.541377\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Best Accuracy on Test data: 0.7920954823493958\n",
      "Current Layer Attributes - epochs:15 hidden layers:1 features count:48\n",
      "Step 1 | Training Loss: 0.000368 | Validation Accuracy: 0.914864\n",
      "Accuracy on Test data: 0.8122338652610779, 0.647426187992096\n",
      "Step 2 | Training Loss: 1.315042 | Validation Accuracy: 0.871602\n",
      "Accuracy on Test data: 0.7071061134338379, 0.4583122432231903\n",
      "Step 3 | Training Loss: 0.944233 | Validation Accuracy: 0.538400\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 4 | Training Loss: 1.076023 | Validation Accuracy: 0.542766\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 0.046245 | Validation Accuracy: 0.518357\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 6 | Training Loss: 0.521358 | Validation Accuracy: 0.529272\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 7 | Training Loss: 0.107319 | Validation Accuracy: 0.526295\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 8 | Training Loss: 1.585423 | Validation Accuracy: 0.540187\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 9 | Training Loss: 1.665708 | Validation Accuracy: 0.530661\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 10 | Training Loss: 1.072209 | Validation Accuracy: 0.521334\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 11 | Training Loss: 0.860059 | Validation Accuracy: 0.529867\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 12 | Training Loss: 1.319000 | Validation Accuracy: 0.541576\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 13 | Training Loss: 0.366902 | Validation Accuracy: 0.537607\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 14 | Training Loss: 1.190232 | Validation Accuracy: 0.525898\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 15 | Training Loss: 0.362021 | Validation Accuracy: 0.536019\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 1 | Training Loss: 0.718000 | Validation Accuracy: 0.528875\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 2 | Training Loss: 1.052636 | Validation Accuracy: 0.531653\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 3 | Training Loss: 0.863017 | Validation Accuracy: 0.537210\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 4 | Training Loss: 2.786286 | Validation Accuracy: 0.537011\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 0.742634 | Validation Accuracy: 0.540583\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 6 | Training Loss: 1.083127 | Validation Accuracy: 0.531852\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 7 | Training Loss: 0.426010 | Validation Accuracy: 0.530859\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 8 | Training Loss: 0.333158 | Validation Accuracy: 0.537210\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 9 | Training Loss: 0.448228 | Validation Accuracy: 0.531455\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 10 | Training Loss: 1.253593 | Validation Accuracy: 0.528279\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 11 | Training Loss: 0.744492 | Validation Accuracy: 0.546537\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 12 | Training Loss: 0.139003 | Validation Accuracy: 0.531852\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 13 | Training Loss: 1.035890 | Validation Accuracy: 0.536614\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 14 | Training Loss: 3.021080 | Validation Accuracy: 0.522723\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 15 | Training Loss: 0.213813 | Validation Accuracy: 0.527684\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Best Accuracy on Test data: 0.8122338652610779\n",
      "Current Layer Attributes - epochs:15 hidden layers:1 features count:122\n",
      "Step 1 | Training Loss: 0.763570 | Validation Accuracy: 0.537011\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 2 | Training Loss: 0.756096 | Validation Accuracy: 0.537011\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 3 | Training Loss: 0.130683 | Validation Accuracy: 0.533638\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 4 | Training Loss: 1.112396 | Validation Accuracy: 0.533241\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 2.970290 | Validation Accuracy: 0.537011\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 6 | Training Loss: 0.214373 | Validation Accuracy: 0.529470\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 7 | Training Loss: 1.212557 | Validation Accuracy: 0.531256\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 8 | Training Loss: 1.729573 | Validation Accuracy: 0.540583\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 9 | Training Loss: 0.746703 | Validation Accuracy: 0.527883\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 10 | Training Loss: 1.527806 | Validation Accuracy: 0.533042\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 11 | Training Loss: 0.094671 | Validation Accuracy: 0.536218\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 12 | Training Loss: 0.807922 | Validation Accuracy: 0.544949\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 13 | Training Loss: 0.894072 | Validation Accuracy: 0.540980\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 14 | Training Loss: 0.135036 | Validation Accuracy: 0.523318\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 15 | Training Loss: 0.040706 | Validation Accuracy: 0.540583\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 1 | Training Loss: 0.391631 | Validation Accuracy: 0.547132\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 2 | Training Loss: 1.237380 | Validation Accuracy: 0.542370\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 3 | Training Loss: 0.263388 | Validation Accuracy: 0.527287\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 4 | Training Loss: 0.399686 | Validation Accuracy: 0.541179\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 0.961852 | Validation Accuracy: 0.535225\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 6 | Training Loss: 0.705434 | Validation Accuracy: 0.537210\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 7 | Training Loss: 0.979439 | Validation Accuracy: 0.543163\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 8 | Training Loss: 0.680752 | Validation Accuracy: 0.538004\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 9 | Training Loss: 0.936349 | Validation Accuracy: 0.544949\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 10 | Training Loss: 0.199504 | Validation Accuracy: 0.539988\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 11 | Training Loss: 0.096401 | Validation Accuracy: 0.543759\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 12 | Training Loss: 0.173291 | Validation Accuracy: 0.532645\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 13 | Training Loss: 0.421284 | Validation Accuracy: 0.543362\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 14 | Training Loss: 1.940808 | Validation Accuracy: 0.537805\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 15 | Training Loss: 0.315776 | Validation Accuracy: 0.530661\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Best Accuracy on Test data: 0.43075764179229736\n",
      "Current Layer Attributes - epochs:15 hidden layers:3 features count:1\n",
      "Step 1 | Training Loss: 0.000093 | Validation Accuracy: 0.643580\n",
      "Accuracy on Test data: 0.6314318776130676, 0.5137552618980408\n",
      "Step 2 | Training Loss: 0.000737 | Validation Accuracy: 0.597539\n",
      "Accuracy on Test data: 0.595901370048523, 0.5198312401771545\n",
      "Step 3 | Training Loss: 0.000062 | Validation Accuracy: 0.626910\n",
      "Accuracy on Test data: 0.6588449478149414, 0.645316481590271\n",
      "Step 4 | Training Loss: 0.000216 | Validation Accuracy: 0.696765\n",
      "Accuracy on Test data: 0.7372693419456482, 0.6072573661804199\n",
      "Step 5 | Training Loss: 0.000007 | Validation Accuracy: 0.725144\n",
      "Accuracy on Test data: 0.7258250713348389, 0.6274261474609375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 6 | Training Loss: 0.000046 | Validation Accuracy: 0.668783\n",
      "Accuracy on Test data: 0.6443843245506287, 0.46928268671035767\n",
      "Step 7 | Training Loss: 0.000046 | Validation Accuracy: 0.681484\n",
      "Accuracy on Test data: 0.6622604727745056, 0.5103797316551208\n",
      "Step 8 | Training Loss: 0.000096 | Validation Accuracy: 0.667196\n",
      "Accuracy on Test data: 0.6324077248573303, 0.47578057646751404\n",
      "Step 9 | Training Loss: 0.000156 | Validation Accuracy: 0.694979\n",
      "Accuracy on Test data: 0.7618878483772278, 0.647763729095459\n",
      "Step 10 | Training Loss: 0.000062 | Validation Accuracy: 0.709069\n",
      "Accuracy on Test data: 0.7138928174972534, 0.5889451503753662\n",
      "Step 11 | Training Loss: 0.000121 | Validation Accuracy: 0.737249\n",
      "Accuracy on Test data: 0.7503548860549927, 0.6370463967323303\n",
      "Step 12 | Training Loss: 0.000149 | Validation Accuracy: 0.706489\n",
      "Accuracy on Test data: 0.697613537311554, 0.5612658262252808\n",
      "Step 13 | Training Loss: 0.000105 | Validation Accuracy: 0.673348\n",
      "Accuracy on Test data: 0.6672285199165344, 0.5334177017211914\n",
      "Step 14 | Training Loss: 0.000102 | Validation Accuracy: 0.674737\n",
      "Accuracy on Test data: 0.674237072467804, 0.5168776512145996\n",
      "Step 15 | Training Loss: 0.000025 | Validation Accuracy: 0.688232\n",
      "Accuracy on Test data: 0.6826649904251099, 0.5340084433555603\n",
      "Step 1 | Training Loss: 0.000021 | Validation Accuracy: 0.725541\n",
      "Accuracy on Test data: 0.7473385334014893, 0.6248100996017456\n",
      "Step 2 | Training Loss: 0.000006 | Validation Accuracy: 0.721969\n",
      "Accuracy on Test data: 0.7300834059715271, 0.6056540012359619\n",
      "Step 3 | Training Loss: 0.000058 | Validation Accuracy: 0.715420\n",
      "Accuracy on Test data: 0.7372693419456482, 0.6158649921417236\n",
      "Step 4 | Training Loss: 0.000099 | Validation Accuracy: 0.697956\n",
      "Accuracy on Test data: 0.7182399034500122, 0.5803375244140625\n",
      "Step 5 | Training Loss: 0.000006 | Validation Accuracy: 0.724549\n",
      "Accuracy on Test data: 0.7513307332992554, 0.6348523497581482\n",
      "Step 6 | Training Loss: 0.000114 | Validation Accuracy: 0.714626\n",
      "Accuracy on Test data: 0.7411284446716309, 0.6107172966003418\n",
      "Step 7 | Training Loss: 0.000177 | Validation Accuracy: 0.723755\n",
      "Accuracy on Test data: 0.7515525221824646, 0.6207594871520996\n",
      "Step 8 | Training Loss: 0.000006 | Validation Accuracy: 0.671562\n",
      "Accuracy on Test data: 0.6394606232643127, 0.47130802273750305\n",
      "Step 9 | Training Loss: 0.000023 | Validation Accuracy: 0.692201\n",
      "Accuracy on Test data: 0.6698899865150452, 0.5066666603088379\n",
      "Step 10 | Training Loss: 0.000054 | Validation Accuracy: 0.692399\n",
      "Accuracy on Test data: 0.6977022886276245, 0.5557805895805359\n",
      "Step 11 | Training Loss: 0.000069 | Validation Accuracy: 0.725938\n",
      "Accuracy on Test data: 0.7234297394752502, 0.5946835279464722\n",
      "Step 12 | Training Loss: 0.000110 | Validation Accuracy: 0.708474\n",
      "Accuracy on Test data: 0.7145581841468811, 0.5817721486091614\n",
      "Step 13 | Training Loss: 0.000157 | Validation Accuracy: 0.739829\n",
      "Accuracy on Test data: 0.7646380662918091, 0.6535021066665649\n",
      "Step 14 | Training Loss: 0.000051 | Validation Accuracy: 0.690415\n",
      "Accuracy on Test data: 0.6781848669052124, 0.5355274081230164\n",
      "Step 15 | Training Loss: 0.000283 | Validation Accuracy: 0.792221\n",
      "Accuracy on Test data: 0.7861515283584595, 0.6638818383216858\n",
      "Best Accuracy on Test data: 0.7861515283584595\n",
      "Current Layer Attributes - epochs:15 hidden layers:3 features count:12\n",
      "Step 1 | Training Loss: 0.000279 | Validation Accuracy: 0.895813\n",
      "Accuracy on Test data: 0.7882806658744812, 0.6433755159378052\n",
      "Step 2 | Training Loss: 0.000117 | Validation Accuracy: 0.868029\n",
      "Accuracy on Test data: 0.7578069567680359, 0.5910548567771912\n",
      "Step 3 | Training Loss: 0.000168 | Validation Accuracy: 0.913276\n",
      "Accuracy on Test data: 0.8248314261436462, 0.6795780658721924\n",
      "Step 4 | Training Loss: 0.000052 | Validation Accuracy: 0.848978\n",
      "Accuracy on Test data: 0.790587306022644, 0.6556962132453918\n",
      "Step 5 | Training Loss: 0.000049 | Validation Accuracy: 0.808891\n",
      "Accuracy on Test data: 0.8168026804924011, 0.699578046798706\n",
      "Step 6 | Training Loss: 0.000156 | Validation Accuracy: 0.810677\n",
      "Accuracy on Test data: 0.8396469354629517, 0.7426160573959351\n",
      "Step 7 | Training Loss: 0.000006 | Validation Accuracy: 0.806311\n",
      "Accuracy on Test data: 0.8388041257858276, 0.7439662218093872\n",
      "Step 8 | Training Loss: 0.000038 | Validation Accuracy: 0.819607\n",
      "Accuracy on Test data: 0.8314407467842102, 0.7313923835754395\n",
      "Step 9 | Training Loss: 0.000054 | Validation Accuracy: 0.807501\n",
      "Accuracy on Test data: 0.7965312004089355, 0.6744303703308105\n",
      "Step 10 | Training Loss: 0.000009 | Validation Accuracy: 0.806311\n",
      "Accuracy on Test data: 0.8097498416900635, 0.7119831442832947\n",
      "Step 11 | Training Loss: 0.000135 | Validation Accuracy: 0.778329\n",
      "Accuracy on Test data: 0.8306422829627991, 0.7534177303314209\n",
      "Step 12 | Training Loss: 0.000137 | Validation Accuracy: 0.797182\n",
      "Accuracy on Test data: 0.8293559551239014, 0.7227004170417786\n",
      "Step 13 | Training Loss: 0.000058 | Validation Accuracy: 0.806112\n",
      "Accuracy on Test data: 0.8036284446716309, 0.6767088770866394\n",
      "Step 14 | Training Loss: 0.000061 | Validation Accuracy: 0.789045\n",
      "Accuracy on Test data: 0.8027856349945068, 0.6749367117881775\n",
      "Step 15 | Training Loss: 0.000001 | Validation Accuracy: 0.797182\n",
      "Accuracy on Test data: 0.8103708028793335, 0.6791561245918274\n",
      "Step 1 | Training Loss: 0.000009 | Validation Accuracy: 0.798373\n",
      "Accuracy on Test data: 0.8103264570236206, 0.6867510676383972\n",
      "Step 2 | Training Loss: 0.000094 | Validation Accuracy: 0.793411\n",
      "Accuracy on Test data: 0.8203956484794617, 0.700843870639801\n",
      "Step 3 | Training Loss: 0.000007 | Validation Accuracy: 0.794007\n",
      "Accuracy on Test data: 0.813653290271759, 0.6978058815002441\n",
      "Step 4 | Training Loss: 0.000047 | Validation Accuracy: 0.806112\n",
      "Accuracy on Test data: 0.824609637260437, 0.7148523330688477\n",
      "Step 5 | Training Loss: 0.000039 | Validation Accuracy: 0.786862\n",
      "Accuracy on Test data: 0.81538325548172, 0.6918143630027771\n",
      "Step 6 | Training Loss: 0.000129 | Validation Accuracy: 0.808891\n",
      "Accuracy on Test data: 0.7985717058181763, 0.6779747009277344\n",
      "Step 7 | Training Loss: 0.000000 | Validation Accuracy: 0.822385\n",
      "Accuracy on Test data: 0.8220812678337097, 0.7082700133323669\n",
      "Step 8 | Training Loss: 0.000009 | Validation Accuracy: 0.722564\n",
      "Accuracy on Test data: 0.7679648399353027, 0.6787341833114624\n",
      "Step 9 | Training Loss: 0.000016 | Validation Accuracy: 0.787061\n",
      "Accuracy on Test data: 0.8216376900672913, 0.7361181378364563\n",
      "Step 10 | Training Loss: 0.000102 | Validation Accuracy: 0.770193\n",
      "Accuracy on Test data: 0.8221699595451355, 0.7306329011917114\n",
      "Step 11 | Training Loss: 0.000045 | Validation Accuracy: 0.708275\n",
      "Accuracy on Test data: 0.7452980875968933, 0.6262447237968445\n",
      "Step 12 | Training Loss: 0.000043 | Validation Accuracy: 0.707879\n",
      "Accuracy on Test data: 0.7305713295936584, 0.6079325079917908\n",
      "Step 13 | Training Loss: 0.000001 | Validation Accuracy: 0.756102\n",
      "Accuracy on Test data: 0.7484918236732483, 0.6216877698898315\n",
      "Step 14 | Training Loss: 0.000008 | Validation Accuracy: 0.765033\n",
      "Accuracy on Test data: 0.805890679359436, 0.7077637314796448\n",
      "Step 15 | Training Loss: 0.000051 | Validation Accuracy: 0.783092\n",
      "Accuracy on Test data: 0.8480305075645447, 0.7746835350990295\n",
      "Best Accuracy on Test data: 0.8480305075645447\n",
      "Current Layer Attributes - epochs:15 hidden layers:3 features count:24\n",
      "Step 1 | Training Loss: 0.000086 | Validation Accuracy: 0.868029\n",
      "Accuracy on Test data: 0.7796309590339661, 0.6357805728912354\n",
      "Step 2 | Training Loss: 0.000111 | Validation Accuracy: 0.894423\n",
      "Accuracy on Test data: 0.8007451891899109, 0.6647257208824158\n",
      "Step 3 | Training Loss: 0.000183 | Validation Accuracy: 0.909506\n",
      "Accuracy on Test data: 0.8608055114746094, 0.7559493780136108\n",
      "Step 4 | Training Loss: 0.000109 | Validation Accuracy: 0.901171\n",
      "Accuracy on Test data: 0.8163147568702698, 0.6899577975273132\n",
      "Step 5 | Training Loss: 1.435426 | Validation Accuracy: 0.845406\n",
      "Accuracy on Test data: 0.7838005423545837, 0.6617721319198608\n",
      "Step 6 | Training Loss: 0.000046 | Validation Accuracy: 0.825362\n",
      "Accuracy on Test data: 0.7087916731834412, 0.6480168700218201\n",
      "Step 7 | Training Loss: 0.000027 | Validation Accuracy: 0.815638\n",
      "Accuracy on Test data: 0.7389993071556091, 0.6801687479019165\n",
      "Step 8 | Training Loss: 0.000091 | Validation Accuracy: 0.858702\n",
      "Accuracy on Test data: 0.7409954071044922, 0.6364557147026062\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 9 | Training Loss: 0.000141 | Validation Accuracy: 0.903751\n",
      "Accuracy on Test data: 0.8419091701507568, 0.751139223575592\n",
      "Step 10 | Training Loss: 0.000025 | Validation Accuracy: 0.926771\n",
      "Accuracy on Test data: 0.8447036743164062, 0.7358649969100952\n",
      "Step 11 | Training Loss: 0.000121 | Validation Accuracy: 0.906331\n",
      "Accuracy on Test data: 0.8088626861572266, 0.7099577784538269\n",
      "Step 12 | Training Loss: 0.000034 | Validation Accuracy: 0.915261\n",
      "Accuracy on Test data: 0.7733321785926819, 0.594936728477478\n",
      "Step 13 | Training Loss: 0.000092 | Validation Accuracy: 0.917444\n",
      "Accuracy on Test data: 0.8308640718460083, 0.6921519041061401\n",
      "Step 14 | Training Loss: 0.000078 | Validation Accuracy: 0.914666\n",
      "Accuracy on Test data: 0.8145847916603088, 0.6745991706848145\n",
      "Step 15 | Training Loss: 0.000006 | Validation Accuracy: 0.928557\n",
      "Accuracy on Test data: 0.7756831049919128, 0.6475949287414551\n",
      "Step 1 | Training Loss: 0.000014 | Validation Accuracy: 0.938678\n",
      "Accuracy on Test data: 0.7747959494590759, 0.5864135026931763\n",
      "Step 2 | Training Loss: 0.000099 | Validation Accuracy: 0.898393\n",
      "Accuracy on Test data: 0.7079932689666748, 0.4611814320087433\n",
      "Step 3 | Training Loss: 0.000020 | Validation Accuracy: 0.932526\n",
      "Accuracy on Test data: 0.7804737687110901, 0.5898734331130981\n",
      "Step 4 | Training Loss: 0.000022 | Validation Accuracy: 0.927763\n",
      "Accuracy on Test data: 0.7914301156997681, 0.6102953553199768\n",
      "Step 5 | Training Loss: 0.000099 | Validation Accuracy: 0.926771\n",
      "Accuracy on Test data: 0.7542139887809753, 0.545485258102417\n",
      "Step 6 | Training Loss: 0.000002 | Validation Accuracy: 0.906331\n",
      "Accuracy on Test data: 0.7510645985603333, 0.5399156212806702\n",
      "Step 7 | Training Loss: 0.000018 | Validation Accuracy: 0.918833\n",
      "Accuracy on Test data: 0.7733321785926819, 0.5786497592926025\n",
      "Step 8 | Training Loss: 0.000089 | Validation Accuracy: 0.916650\n",
      "Accuracy on Test data: 0.761799156665802, 0.5546835660934448\n",
      "Step 9 | Training Loss: 0.000116 | Validation Accuracy: 0.910300\n",
      "Accuracy on Test data: 0.7876153588294983, 0.603966236114502\n",
      "Step 10 | Training Loss: 0.000036 | Validation Accuracy: 0.913673\n",
      "Accuracy on Test data: 0.7576295137405396, 0.5553586483001709\n",
      "Step 11 | Training Loss: 0.000127 | Validation Accuracy: 0.903552\n",
      "Accuracy on Test data: 0.7853087186813354, 0.605991542339325\n",
      "Step 12 | Training Loss: 0.000209 | Validation Accuracy: 0.900972\n",
      "Accuracy on Test data: 0.754347026348114, 0.5496202707290649\n",
      "Step 13 | Training Loss: 0.000066 | Validation Accuracy: 0.875967\n",
      "Accuracy on Test data: 0.7751508355140686, 0.5967932343482971\n",
      "Step 14 | Training Loss: 0.000044 | Validation Accuracy: 0.905537\n",
      "Accuracy on Test data: 0.7607789039611816, 0.556033730506897\n",
      "Step 15 | Training Loss: 0.000064 | Validation Accuracy: 0.902362\n",
      "Accuracy on Test data: 0.7811391353607178, 0.5908860564231873\n",
      "Best Accuracy on Test data: 0.8608055114746094\n",
      "Current Layer Attributes - epochs:15 hidden layers:3 features count:48\n",
      "Step 1 | Training Loss: 0.000347 | Validation Accuracy: 0.884501\n",
      "Accuracy on Test data: 0.8092618584632874, 0.6761181354522705\n",
      "Step 2 | Training Loss: 0.000208 | Validation Accuracy: 0.904346\n",
      "Accuracy on Test data: 0.8415099382400513, 0.7249789237976074\n",
      "Step 3 | Training Loss: 0.000039 | Validation Accuracy: 0.887081\n",
      "Accuracy on Test data: 0.8492282032966614, 0.7174683809280396\n",
      "Step 4 | Training Loss: 0.000073 | Validation Accuracy: 0.897202\n",
      "Accuracy on Test data: 0.8461675047874451, 0.7134177088737488\n",
      "Step 5 | Training Loss: 0.000024 | Validation Accuracy: 0.895217\n",
      "Accuracy on Test data: 0.8549503087997437, 0.7382278442382812\n",
      "Step 6 | Training Loss: 0.000026 | Validation Accuracy: 0.892042\n",
      "Accuracy on Test data: 0.8667938113212585, 0.7738396525382996\n",
      "Step 7 | Training Loss: 0.000022 | Validation Accuracy: 0.886485\n",
      "Accuracy on Test data: 0.8884403705596924, 0.801518976688385\n",
      "Step 8 | Training Loss: 0.000046 | Validation Accuracy: 0.883707\n",
      "Accuracy on Test data: 0.8852466344833374, 0.8017721772193909\n",
      "Step 9 | Training Loss: 0.000068 | Validation Accuracy: 0.894423\n",
      "Accuracy on Test data: 0.8867548108100891, 0.795443058013916\n",
      "Step 10 | Training Loss: 0.000186 | Validation Accuracy: 0.846993\n",
      "Accuracy on Test data: 0.8771735429763794, 0.8194093108177185\n",
      "Step 11 | Training Loss: 0.000033 | Validation Accuracy: 0.870609\n",
      "Accuracy on Test data: 0.8665720224380493, 0.8086919784545898\n",
      "Step 12 | Training Loss: 0.000070 | Validation Accuracy: 0.818615\n",
      "Accuracy on Test data: 0.8417760729789734, 0.7779746651649475\n",
      "Step 13 | Training Loss: 0.000002 | Validation Accuracy: 0.835483\n",
      "Accuracy on Test data: 0.8708747625350952, 0.8137552738189697\n",
      "Step 14 | Training Loss: 0.000024 | Validation Accuracy: 0.837865\n",
      "Accuracy on Test data: 0.8723385334014893, 0.8140928149223328\n",
      "Step 15 | Training Loss: 0.000037 | Validation Accuracy: 0.838658\n",
      "Accuracy on Test data: 0.8765525221824646, 0.8258227705955505\n",
      "Step 1 | Training Loss: 0.000087 | Validation Accuracy: 0.866640\n",
      "Accuracy on Test data: 0.8734918236732483, 0.7939240336418152\n",
      "Step 2 | Training Loss: 0.000059 | Validation Accuracy: 0.865053\n",
      "Accuracy on Test data: 0.8814762234687805, 0.8101266026496887\n",
      "Step 3 | Training Loss: 0.000072 | Validation Accuracy: 0.766223\n",
      "Accuracy on Test data: 0.840445339679718, 0.7813501954078674\n",
      "Step 4 | Training Loss: 0.000023 | Validation Accuracy: 0.785275\n",
      "Accuracy on Test data: 0.844481885433197, 0.7789029479026794\n",
      "Step 5 | Training Loss: 0.000051 | Validation Accuracy: 0.813257\n",
      "Accuracy on Test data: 0.8501153588294983, 0.7616033554077148\n",
      "Step 6 | Training Loss: 0.000136 | Validation Accuracy: 0.817226\n",
      "Accuracy on Test data: 0.8500709533691406, 0.7587341666221619\n",
      "Step 7 | Training Loss: 0.000059 | Validation Accuracy: 0.829133\n",
      "Accuracy on Test data: 0.8434173464775085, 0.7453164458274841\n",
      "Step 8 | Training Loss: 0.000060 | Validation Accuracy: 0.841238\n",
      "Accuracy on Test data: 0.8391146063804626, 0.7361181378364563\n",
      "Step 9 | Training Loss: 0.000038 | Validation Accuracy: 0.797579\n",
      "Accuracy on Test data: 0.8138307332992554, 0.6846413612365723\n",
      "Step 10 | Training Loss: 0.000049 | Validation Accuracy: 0.796785\n",
      "Accuracy on Test data: 0.805890679359436, 0.6732489466667175\n",
      "Step 11 | Training Loss: 0.000072 | Validation Accuracy: 0.776940\n",
      "Accuracy on Test data: 0.8035397529602051, 0.6749367117881775\n",
      "Step 12 | Training Loss: 0.000030 | Validation Accuracy: 0.804326\n",
      "Accuracy on Test data: 0.8095280528068542, 0.6799156069755554\n",
      "Step 13 | Training Loss: 0.000085 | Validation Accuracy: 0.803334\n",
      "Accuracy on Test data: 0.8109474778175354, 0.6880168914794922\n",
      "Step 14 | Training Loss: 0.000031 | Validation Accuracy: 0.806112\n",
      "Accuracy on Test data: 0.825141966342926, 0.7178903222084045\n",
      "Step 15 | Training Loss: 0.000110 | Validation Accuracy: 0.799960\n",
      "Accuracy on Test data: 0.8295333385467529, 0.7158649563789368\n",
      "Best Accuracy on Test data: 0.8884403705596924\n",
      "Current Layer Attributes - epochs:15 hidden layers:3 features count:122\n",
      "Step 1 | Training Loss: 0.000374 | Validation Accuracy: 0.877357\n",
      "Accuracy on Test data: 0.7500443458557129, 0.5751898884773254\n",
      "Step 2 | Training Loss: 0.470249 | Validation Accuracy: 0.535027\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 3 | Training Loss: 0.291069 | Validation Accuracy: 0.553880\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 4 | Training Loss: 0.748151 | Validation Accuracy: 0.533439\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 0.128098 | Validation Accuracy: 0.535622\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 6 | Training Loss: 0.591737 | Validation Accuracy: 0.541576\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 7 | Training Loss: 0.140650 | Validation Accuracy: 0.529669\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 8 | Training Loss: 0.280989 | Validation Accuracy: 0.533042\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 9 | Training Loss: 0.040990 | Validation Accuracy: 0.525700\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 10 | Training Loss: 1.556867 | Validation Accuracy: 0.538004\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 11 | Training Loss: 0.118808 | Validation Accuracy: 0.521334\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 12 | Training Loss: 0.178845 | Validation Accuracy: 0.527883\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 13 | Training Loss: 0.798469 | Validation Accuracy: 0.531852\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 14 | Training Loss: 0.688534 | Validation Accuracy: 0.539988\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 15 | Training Loss: 0.495494 | Validation Accuracy: 0.528875\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 1 | Training Loss: 0.170531 | Validation Accuracy: 0.538004\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 2 | Training Loss: 0.804761 | Validation Accuracy: 0.531256\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 3 | Training Loss: 0.115751 | Validation Accuracy: 0.543362\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 4 | Training Loss: 0.866743 | Validation Accuracy: 0.529867\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 1.777332 | Validation Accuracy: 0.524509\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 6 | Training Loss: 0.207653 | Validation Accuracy: 0.524906\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 7 | Training Loss: 0.018124 | Validation Accuracy: 0.536614\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 8 | Training Loss: 0.531272 | Validation Accuracy: 0.533439\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 9 | Training Loss: 0.878964 | Validation Accuracy: 0.531256\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 10 | Training Loss: 0.343119 | Validation Accuracy: 0.536813\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 11 | Training Loss: 0.019428 | Validation Accuracy: 0.534828\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 12 | Training Loss: 0.266668 | Validation Accuracy: 0.526493\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 13 | Training Loss: 0.947765 | Validation Accuracy: 0.531455\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 14 | Training Loss: 1.278037 | Validation Accuracy: 0.535424\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 15 | Training Loss: 0.399154 | Validation Accuracy: 0.532844\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Best Accuracy on Test data: 0.7500443458557129\n"
     ]
    }
   ],
   "source": [
    "#%%timeit -r 10\n",
    "\n",
    "Hyperparameters.start_training()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:41:04.409623Z",
     "start_time": "2017-07-23T21:41:04.337474Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(Train.predictions).to_pickle(\"dataset/tf_vae_dense_trained_together_nsl_kdd_predictions-.pkl\")\n",
    "pd.Panel(Train.predictions_).to_pickle(\"dataset/tf_vae_dense_trained_together_nsl_kdd_predictions-__.pkl\")\n",
    "df_results.to_pickle(\"dataset/tf_vae_dense_trained_together_nsl_kdd_scores-.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:41:04.489405Z",
     "start_time": "2017-07-23T21:41:04.411433Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    np.set_printoptions(precision=4)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        #print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        #print('Confusion matrix, without normalization')\n",
    "        pass\n",
    "    \n",
    "    #print(cm)\n",
    "\n",
    "    label = [[\"\\n True Negative\", \"\\n False Positive \\n Type II Error\"],\n",
    "             [\"\\n False Negative \\n Type I Error\", \"\\n True Positive\"]\n",
    "            ]\n",
    "    \n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        \n",
    "        plt.text(j, i, \"{} {}\".format(cm[i, j].round(4), label[i][j]),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "def plot(actual_value, pred_value):\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "\n",
    "    cm_2labels = confusion_matrix(y_pred = pred_value, y_true = actual_value)\n",
    "    plt.figure(figsize=[6,6])\n",
    "    plot_confusion_matrix(cm_2labels, ['Normal', 'Attack'], normalize = False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:41:04.497959Z",
     "start_time": "2017-07-23T21:41:04.490881Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "past_scores = pd.read_pickle(\"dataset/scores/tf_vae_dense_trained_together_nsl_kdd_all-.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:41:04.521047Z",
     "start_time": "2017-07-23T21:41:04.499421Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>f1_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>8</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.886485</td>\n",
       "      <td>0.888440</td>\n",
       "      <td>0.897502</td>\n",
       "      <td>0.801519</td>\n",
       "      <td>0.870041</td>\n",
       "      <td>10.408620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>7</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.892042</td>\n",
       "      <td>0.866794</td>\n",
       "      <td>0.879101</td>\n",
       "      <td>0.773840</td>\n",
       "      <td>0.853839</td>\n",
       "      <td>8.919692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>32</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.783092</td>\n",
       "      <td>0.848031</td>\n",
       "      <td>0.874127</td>\n",
       "      <td>0.774684</td>\n",
       "      <td>0.869207</td>\n",
       "      <td>41.830300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>4</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.909506</td>\n",
       "      <td>0.860806</td>\n",
       "      <td>0.871478</td>\n",
       "      <td>0.755949</td>\n",
       "      <td>0.839262</td>\n",
       "      <td>4.260230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>6</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.895217</td>\n",
       "      <td>0.854950</td>\n",
       "      <td>0.863829</td>\n",
       "      <td>0.738228</td>\n",
       "      <td>0.823750</td>\n",
       "      <td>7.443657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.810677</td>\n",
       "      <td>0.839647</td>\n",
       "      <td>0.862595</td>\n",
       "      <td>0.742616</td>\n",
       "      <td>0.844277</td>\n",
       "      <td>8.260208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>4</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.887081</td>\n",
       "      <td>0.849228</td>\n",
       "      <td>0.861044</td>\n",
       "      <td>0.717468</td>\n",
       "      <td>0.815130</td>\n",
       "      <td>4.494314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>3</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.904346</td>\n",
       "      <td>0.841510</td>\n",
       "      <td>0.847977</td>\n",
       "      <td>0.724979</td>\n",
       "      <td>0.807672</td>\n",
       "      <td>3.025846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.913276</td>\n",
       "      <td>0.824831</td>\n",
       "      <td>0.831764</td>\n",
       "      <td>0.679578</td>\n",
       "      <td>0.778562</td>\n",
       "      <td>4.172075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2</td>\n",
       "      <td>48</td>\n",
       "      <td>1</td>\n",
       "      <td>0.914864</td>\n",
       "      <td>0.812234</td>\n",
       "      <td>0.815209</td>\n",
       "      <td>0.647426</td>\n",
       "      <td>0.748767</td>\n",
       "      <td>1.302569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.884501</td>\n",
       "      <td>0.809262</td>\n",
       "      <td>0.811470</td>\n",
       "      <td>0.676118</td>\n",
       "      <td>0.768739</td>\n",
       "      <td>1.547668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.792221</td>\n",
       "      <td>0.786152</td>\n",
       "      <td>0.800149</td>\n",
       "      <td>0.663882</td>\n",
       "      <td>0.778192</td>\n",
       "      <td>39.101900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.894423</td>\n",
       "      <td>0.800745</td>\n",
       "      <td>0.799249</td>\n",
       "      <td>0.664726</td>\n",
       "      <td>0.758993</td>\n",
       "      <td>2.863533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.773963</td>\n",
       "      <td>0.789700</td>\n",
       "      <td>0.790397</td>\n",
       "      <td>0.708186</td>\n",
       "      <td>0.800092</td>\n",
       "      <td>2.712769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.895813</td>\n",
       "      <td>0.788281</td>\n",
       "      <td>0.784796</td>\n",
       "      <td>0.643376</td>\n",
       "      <td>0.739104</td>\n",
       "      <td>1.435609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0.864854</td>\n",
       "      <td>0.792095</td>\n",
       "      <td>0.784297</td>\n",
       "      <td>0.636118</td>\n",
       "      <td>0.725035</td>\n",
       "      <td>2.065240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>28</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.739829</td>\n",
       "      <td>0.764638</td>\n",
       "      <td>0.781969</td>\n",
       "      <td>0.653502</td>\n",
       "      <td>0.774470</td>\n",
       "      <td>36.502404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.868029</td>\n",
       "      <td>0.779631</td>\n",
       "      <td>0.779082</td>\n",
       "      <td>0.635781</td>\n",
       "      <td>0.738962</td>\n",
       "      <td>1.464846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.694979</td>\n",
       "      <td>0.761888</td>\n",
       "      <td>0.773349</td>\n",
       "      <td>0.647764</td>\n",
       "      <td>0.759118</td>\n",
       "      <td>11.878906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0.815241</td>\n",
       "      <td>0.748802</td>\n",
       "      <td>0.741640</td>\n",
       "      <td>0.569114</td>\n",
       "      <td>0.676876</td>\n",
       "      <td>0.853013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.696765</td>\n",
       "      <td>0.737269</td>\n",
       "      <td>0.739109</td>\n",
       "      <td>0.607257</td>\n",
       "      <td>0.723601</td>\n",
       "      <td>5.346412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2</td>\n",
       "      <td>122</td>\n",
       "      <td>3</td>\n",
       "      <td>0.877357</td>\n",
       "      <td>0.750044</td>\n",
       "      <td>0.736818</td>\n",
       "      <td>0.575190</td>\n",
       "      <td>0.660140</td>\n",
       "      <td>1.778840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.652113</td>\n",
       "      <td>0.652280</td>\n",
       "      <td>0.725458</td>\n",
       "      <td>0.692574</td>\n",
       "      <td>0.802301</td>\n",
       "      <td>0.737689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.626910</td>\n",
       "      <td>0.658845</td>\n",
       "      <td>0.691978</td>\n",
       "      <td>0.645316</td>\n",
       "      <td>0.759016</td>\n",
       "      <td>4.035135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.815836</td>\n",
       "      <td>0.716865</td>\n",
       "      <td>0.678130</td>\n",
       "      <td>0.488776</td>\n",
       "      <td>0.559098</td>\n",
       "      <td>1.401122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.643580</td>\n",
       "      <td>0.631432</td>\n",
       "      <td>0.592796</td>\n",
       "      <td>0.513755</td>\n",
       "      <td>0.610518</td>\n",
       "      <td>1.428993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0.871998</td>\n",
       "      <td>0.666164</td>\n",
       "      <td>0.591200</td>\n",
       "      <td>0.452574</td>\n",
       "      <td>0.508151</td>\n",
       "      <td>0.956361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2</td>\n",
       "      <td>122</td>\n",
       "      <td>1</td>\n",
       "      <td>0.537011</td>\n",
       "      <td>0.430758</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.181603</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.642546</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    epoch  no_of_features  hidden_layers  train_score  test_score  f1_score  \\\n",
       "26      8              48              3     0.886485    0.888440  0.897502   \n",
       "25      7              48              3     0.892042    0.866794  0.879101   \n",
       "17     32              12              3     0.783092    0.848031  0.874127   \n",
       "20      4              24              3     0.909506    0.860806  0.871478   \n",
       "24      6              48              3     0.895217    0.854950  0.863829   \n",
       "16      7              12              3     0.810677    0.839647  0.862595   \n",
       "23      4              48              3     0.887081    0.849228  0.861044   \n",
       "22      3              48              3     0.904346    0.841510  0.847977   \n",
       "15      4              12              3     0.913276    0.824831  0.831764   \n",
       "6       2              48              1     0.914864    0.812234  0.815209   \n",
       "21      2              48              3     0.884501    0.809262  0.811470   \n",
       "13     32               1              3     0.792221    0.786152  0.800149   \n",
       "19      3              24              3     0.894423    0.800745  0.799249   \n",
       "2       5               1              1     0.773963    0.789700  0.790397   \n",
       "14      2              12              3     0.895813    0.788281  0.784796   \n",
       "5       3              24              1     0.864854    0.792095  0.784297   \n",
       "12     28               1              3     0.739829    0.764638  0.781969   \n",
       "18      2              24              3     0.868029    0.779631  0.779082   \n",
       "11     10               1              3     0.694979    0.761888  0.773349   \n",
       "3       2              12              1     0.815241    0.748802  0.741640   \n",
       "10      5               1              3     0.696765    0.737269  0.739109   \n",
       "27      2             122              3     0.877357    0.750044  0.736818   \n",
       "0       2               1              1     0.652113    0.652280  0.725458   \n",
       "9       4               1              3     0.626910    0.658845  0.691978   \n",
       "1       3               1              1     0.815836    0.716865  0.678130   \n",
       "8       2               1              3     0.643580    0.631432  0.592796   \n",
       "4       2              24              1     0.871998    0.666164  0.591200   \n",
       "7       2             122              1     0.537011    0.430758  0.000000   \n",
       "\n",
       "    test_score_20  f1_score_20  time_taken  \n",
       "26       0.801519     0.870041   10.408620  \n",
       "25       0.773840     0.853839    8.919692  \n",
       "17       0.774684     0.869207   41.830300  \n",
       "20       0.755949     0.839262    4.260230  \n",
       "24       0.738228     0.823750    7.443657  \n",
       "16       0.742616     0.844277    8.260208  \n",
       "23       0.717468     0.815130    4.494314  \n",
       "22       0.724979     0.807672    3.025846  \n",
       "15       0.679578     0.778562    4.172075  \n",
       "6        0.647426     0.748767    1.302569  \n",
       "21       0.676118     0.768739    1.547668  \n",
       "13       0.663882     0.778192   39.101900  \n",
       "19       0.664726     0.758993    2.863533  \n",
       "2        0.708186     0.800092    2.712769  \n",
       "14       0.643376     0.739104    1.435609  \n",
       "5        0.636118     0.725035    2.065240  \n",
       "12       0.653502     0.774470   36.502404  \n",
       "18       0.635781     0.738962    1.464846  \n",
       "11       0.647764     0.759118   11.878906  \n",
       "3        0.569114     0.676876    0.853013  \n",
       "10       0.607257     0.723601    5.346412  \n",
       "27       0.575190     0.660140    1.778840  \n",
       "0        0.692574     0.802301    0.737689  \n",
       "9        0.645316     0.759016    4.035135  \n",
       "1        0.488776     0.559098    1.401122  \n",
       "8        0.513755     0.610518    1.428993  \n",
       "4        0.452574     0.508151    0.956361  \n",
       "7        0.181603     0.000000    1.642546  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "past_scores.sort_values(by='f1_score',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:41:04.542908Z",
     "start_time": "2017-07-23T21:41:04.522808Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>f1_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>0.886485</td>\n",
       "      <td>0.888440</td>\n",
       "      <td>0.897502</td>\n",
       "      <td>0.801519</td>\n",
       "      <td>0.870041</td>\n",
       "      <td>10.408620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.909506</td>\n",
       "      <td>0.860806</td>\n",
       "      <td>0.871478</td>\n",
       "      <td>0.755949</td>\n",
       "      <td>0.839262</td>\n",
       "      <td>4.260230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <th>3</th>\n",
       "      <td>32</td>\n",
       "      <td>0.783092</td>\n",
       "      <td>0.848031</td>\n",
       "      <td>0.874127</td>\n",
       "      <td>0.774684</td>\n",
       "      <td>0.869207</td>\n",
       "      <td>41.830300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.914864</td>\n",
       "      <td>0.812234</td>\n",
       "      <td>0.815209</td>\n",
       "      <td>0.647426</td>\n",
       "      <td>0.748767</td>\n",
       "      <td>1.302569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>0.864854</td>\n",
       "      <td>0.792095</td>\n",
       "      <td>0.784297</td>\n",
       "      <td>0.636118</td>\n",
       "      <td>0.725035</td>\n",
       "      <td>2.065240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>0.773963</td>\n",
       "      <td>0.789700</td>\n",
       "      <td>0.790397</td>\n",
       "      <td>0.708186</td>\n",
       "      <td>0.800092</td>\n",
       "      <td>2.712769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32</td>\n",
       "      <td>0.792221</td>\n",
       "      <td>0.786152</td>\n",
       "      <td>0.800149</td>\n",
       "      <td>0.663882</td>\n",
       "      <td>0.778192</td>\n",
       "      <td>39.101900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>0.877357</td>\n",
       "      <td>0.750044</td>\n",
       "      <td>0.736818</td>\n",
       "      <td>0.575190</td>\n",
       "      <td>0.660140</td>\n",
       "      <td>1.778840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.815241</td>\n",
       "      <td>0.748802</td>\n",
       "      <td>0.741640</td>\n",
       "      <td>0.569114</td>\n",
       "      <td>0.676876</td>\n",
       "      <td>0.853013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.537011</td>\n",
       "      <td>0.430758</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.181603</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.642546</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              epoch  train_score  test_score  f1_score  \\\n",
       "no_of_features hidden_layers                                             \n",
       "48             3                  8     0.886485    0.888440  0.897502   \n",
       "24             3                  4     0.909506    0.860806  0.871478   \n",
       "12             3                 32     0.783092    0.848031  0.874127   \n",
       "48             1                  2     0.914864    0.812234  0.815209   \n",
       "24             1                  3     0.864854    0.792095  0.784297   \n",
       "1              1                  5     0.773963    0.789700  0.790397   \n",
       "               3                 32     0.792221    0.786152  0.800149   \n",
       "122            3                  2     0.877357    0.750044  0.736818   \n",
       "12             1                  2     0.815241    0.748802  0.741640   \n",
       "122            1                  2     0.537011    0.430758  0.000000   \n",
       "\n",
       "                              test_score_20  f1_score_20  time_taken  \n",
       "no_of_features hidden_layers                                          \n",
       "48             3                   0.801519     0.870041   10.408620  \n",
       "24             3                   0.755949     0.839262    4.260230  \n",
       "12             3                   0.774684     0.869207   41.830300  \n",
       "48             1                   0.647426     0.748767    1.302569  \n",
       "24             1                   0.636118     0.725035    2.065240  \n",
       "1              1                   0.708186     0.800092    2.712769  \n",
       "               3                   0.663882     0.778192   39.101900  \n",
       "122            3                   0.575190     0.660140    1.778840  \n",
       "12             1                   0.569114     0.676876    0.853013  \n",
       "122            1                   0.181603     0.000000    1.642546  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "psg = past_scores.sort_values(by='test_score', ascending=False).groupby(by=['no_of_features', 'hidden_layers'])\n",
    "psg.first().sort_values(by='test_score', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:41:04.560852Z",
     "start_time": "2017-07-23T21:41:04.544258Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>f1_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <th>3</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.891612</td>\n",
       "      <td>0.851697</td>\n",
       "      <td>0.860154</td>\n",
       "      <td>0.738692</td>\n",
       "      <td>0.823195</td>\n",
       "      <td>5.973299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <th>3</th>\n",
       "      <td>11.250000</td>\n",
       "      <td>0.850714</td>\n",
       "      <td>0.825197</td>\n",
       "      <td>0.838321</td>\n",
       "      <td>0.710063</td>\n",
       "      <td>0.807787</td>\n",
       "      <td>13.924548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <th>3</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.890653</td>\n",
       "      <td>0.813727</td>\n",
       "      <td>0.816603</td>\n",
       "      <td>0.685485</td>\n",
       "      <td>0.779072</td>\n",
       "      <td>2.862870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <th>1</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.914864</td>\n",
       "      <td>0.812234</td>\n",
       "      <td>0.815209</td>\n",
       "      <td>0.647426</td>\n",
       "      <td>0.748767</td>\n",
       "      <td>1.302569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <th>3</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.877357</td>\n",
       "      <td>0.750044</td>\n",
       "      <td>0.736818</td>\n",
       "      <td>0.575190</td>\n",
       "      <td>0.660140</td>\n",
       "      <td>1.778840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <th>1</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.815241</td>\n",
       "      <td>0.748802</td>\n",
       "      <td>0.741640</td>\n",
       "      <td>0.569114</td>\n",
       "      <td>0.676876</td>\n",
       "      <td>0.853013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <th>1</th>\n",
       "      <td>2.500000</td>\n",
       "      <td>0.868426</td>\n",
       "      <td>0.729130</td>\n",
       "      <td>0.687749</td>\n",
       "      <td>0.544346</td>\n",
       "      <td>0.616593</td>\n",
       "      <td>1.510800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>3</th>\n",
       "      <td>13.500000</td>\n",
       "      <td>0.699047</td>\n",
       "      <td>0.723371</td>\n",
       "      <td>0.729892</td>\n",
       "      <td>0.621913</td>\n",
       "      <td>0.734153</td>\n",
       "      <td>16.382292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.333333</td>\n",
       "      <td>0.747304</td>\n",
       "      <td>0.719615</td>\n",
       "      <td>0.731329</td>\n",
       "      <td>0.629845</td>\n",
       "      <td>0.720497</td>\n",
       "      <td>1.617193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <th>1</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.537011</td>\n",
       "      <td>0.430758</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.181603</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.642546</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  epoch  train_score  test_score  f1_score  \\\n",
       "no_of_features hidden_layers                                                 \n",
       "48             3               5.000000     0.891612    0.851697  0.860154   \n",
       "12             3              11.250000     0.850714    0.825197  0.838321   \n",
       "24             3               3.000000     0.890653    0.813727  0.816603   \n",
       "48             1               2.000000     0.914864    0.812234  0.815209   \n",
       "122            3               2.000000     0.877357    0.750044  0.736818   \n",
       "12             1               2.000000     0.815241    0.748802  0.741640   \n",
       "24             1               2.500000     0.868426    0.729130  0.687749   \n",
       "1              3              13.500000     0.699047    0.723371  0.729892   \n",
       "               1               3.333333     0.747304    0.719615  0.731329   \n",
       "122            1               2.000000     0.537011    0.430758  0.000000   \n",
       "\n",
       "                              test_score_20  f1_score_20  time_taken  \n",
       "no_of_features hidden_layers                                          \n",
       "48             3                   0.738692     0.823195    5.973299  \n",
       "12             3                   0.710063     0.807787   13.924548  \n",
       "24             3                   0.685485     0.779072    2.862870  \n",
       "48             1                   0.647426     0.748767    1.302569  \n",
       "122            3                   0.575190     0.660140    1.778840  \n",
       "12             1                   0.569114     0.676876    0.853013  \n",
       "24             1                   0.544346     0.616593    1.510800  \n",
       "1              3                   0.621913     0.734153   16.382292  \n",
       "               1                   0.629845     0.720497    1.617193  \n",
       "122            1                   0.181603     0.000000    1.642546  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "psg.mean().sort_values(by='test_score', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:41:04.580103Z",
     "start_time": "2017-07-23T21:41:04.562397Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Train.predictions = pd.read_pickle(\"dataset/tf_vae_dense_trained_together_nsl_kdd_predictions-.pkl\")\n",
    "Train.predictions_ = pd.read_pickle(\"dataset/tf_vae_dense_trained_together_nsl_kdd_predictions-__.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:47:29.547137Z",
     "start_time": "2017-07-23T21:47:29.533501Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Actual</th>\n",
       "      <th>Attack_prob</th>\n",
       "      <th>Normal_prob</th>\n",
       "      <th>Prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11602</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.99998</td>\n",
       "      <td>0.00002</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Actual  Attack_prob  Normal_prob  Prediction\n",
       "11602     0.0      0.99998      0.00002         0.0"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#epoch_nof_hidden\n",
    "Train.predictions[\"8_48_3\"].sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:47:39.870280Z",
     "start_time": "2017-07-23T21:47:39.857356Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Actual</th>\n",
       "      <th>Attack_prob</th>\n",
       "      <th>Normal_prob</th>\n",
       "      <th>Prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>903</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.599516e-07</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Actual   Attack_prob  Normal_prob  Prediction\n",
       "903     1.0  2.599516e-07          1.0         1.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Train.predictions_[\"8_48_3\"].sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:47:47.005483Z",
     "start_time": "2017-07-23T21:47:46.996256Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = Train.predictions[\"8_48_3\"].dropna()\n",
    "df_ = Train.predictions_[\"8_48_3\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:47:48.096198Z",
     "start_time": "2017-07-23T21:47:48.087381Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import metrics as me\n",
    "def get_score(y_true, y_pred):\n",
    "    f1 = me.f1_score(y_true, y_pred)\n",
    "    pre = me.precision_score(y_true, y_pred)\n",
    "    rec = me.recall_score(y_true, y_pred)\n",
    "    acc = me.accuracy_score(y_true, y_pred)\n",
    "    return {\"F1 Score\":f1, \"Precision\":pre, \"Recall\":rec, \"Accuracy\":acc}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:47:50.268967Z",
     "start_time": "2017-07-23T21:47:50.212886Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1 Score</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Scenario</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.888440</td>\n",
       "      <td>0.897502</td>\n",
       "      <td>0.940789</td>\n",
       "      <td>0.858022</td>\n",
       "      <td>Train+/Test+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.801519</td>\n",
       "      <td>0.870041</td>\n",
       "      <td>0.937262</td>\n",
       "      <td>0.811817</td>\n",
       "      <td>Train+/Test-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  F1 Score  Precision    Recall      Scenario\n",
       "0  0.888440  0.897502   0.940789  0.858022  Train+/Test+\n",
       "1  0.801519  0.870041   0.937262  0.811817  Train+/Test-"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import metrics as me\n",
    "\n",
    "scores = get_score(df.loc[:,'Actual'].values.astype(int),\n",
    "                df.loc[:,'Prediction'].values.astype(int))\n",
    "scores.update({\"Scenario\":\"Train+/Test+\"})\n",
    "score_df = pd.DataFrame(scores, index=[0])\n",
    "\n",
    "scores = get_score(df_.loc[:,'Actual'].values.astype(int),\n",
    "                df_.loc[:,'Prediction'].values.astype(int))\n",
    "scores.update({\"Scenario\":\"Train+/Test-\"})\n",
    "\n",
    "score_df = score_df.append(pd.DataFrame(scores, index=[1]))\n",
    "\n",
    "score_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:47:51.203764Z",
     "start_time": "2017-07-23T21:47:51.195502Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Actual\n",
       "0.0     9711\n",
       "1.0    12833\n",
       "Name: Actual, dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(by=\"Actual\").Actual.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:47:53.035678Z",
     "start_time": "2017-07-23T21:47:52.687401Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbYAAAGhCAYAAAAN2pFTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcT+X7x/HXNfYlOykqlELITvaESIhsbVKivX7t9G3R\nItpU2qRVK1qp7Epl30mbCKksaSG74fr9cc5MY5gxmxlzvJ/fx+fR53Of7fr4Dtdc932f+5i7IyIi\nEhUxWR2AiIhIRlJiExGRSFFiExGRSFFiExGRSFFiExGRSFFiExGRSFFiExGRSFFiExGRSFFiExGR\nSMmZ1QGIiEjGyFHoJPfYHRl2Pt/xx0R3b5NhJ8wkSmwiIhHhsTvIc1q3DDvfzsXPlciwk2UiJTYR\nkcgwMI0w6U9AREQiRRWbiEhUGGCW1VFkOSU2EZEoUVekuiJFRCRaVLGJiESJuiKV2EREokOzIkFd\nkSIiEjGq2EREokRdkUpsIiKRYagrEnVFiohIxKhiExGJDFNXJEpsIiLRoq5IdUWKiEi0qGITEYkS\ndUUqsYmIRIdu0AZ1RYqISMSoYhMRiQo9tgZQxSYiIhGjik1EJEo0xqbEJiISHZo8AuqKFBGRiFHF\nJiISJTGaPKLEJiISFVrdH1BXpIiIRIwqNhGRKNF9bEpsIiLRoVmRoK5IERGJGFVsIiJRoq5IJTYR\nkUhRV6S6IkVEJFpUsYmIRIWZuiJRxSYiIhGjik1EJEo0xqaKTUQkUuK6IzPidchL2atmttHMliVo\nK2Zmk83sp/C/RRNs629mK8zsRzM7J0F7bTP7Jtw21Cy4uJnlMbNRYfscMyuXkj8CJTYREUmr14E2\nidr6AVPdvSIwNfyMmVUBegCnh8c8b2Y5wmNeAPoAFcNX3Dl7A3+7+ynAk8AjKQlKiU0iyczymdkn\nZrbZzN5Lx3kuNrNJGRlbVjGzJmb2Y1bHIYdTuPJIRr0Owd2/Av5K1NwRGBG+HwGcn6B9pLvvcvdV\nwAqgnpkdBxRy99nu7sAbiY6JO9f7wNlx1VxylNgkS5nZRWY238y2mtk6MxtvZo0z4NRdgGOB4u7e\nNa0ncfe33b11BsRzWJmZm9kpye3j7l+7+2mZFZNkkYztiiwR/v2Me/VNQQTHuvu68P16gr+HAGWA\ntQn2+zVsKxO+T9y+3zHuHgtsBoofKgBNHpEsY2a3EHRTXA1MBHYD5wAdgOnpPP1JwPLwL8NRz8xy\n6s9C0mCTu9dJ68Hu7mbmGRlQSqhikyxhZoWBB4Dr3P1Dd9/m7nvc/VN3vyPcJ4+ZPWVmv4evp8ws\nT7ituZn9ama3hoPX68zs8nDb/cC9QPewEuxtZgPM7K0E1y8XVjk5w8+9zOxnM/vXzFaZ2cUJ2qcn\nOK6hmc0LuzjnmVnDBNummdmDZjYjPM8kMyuRxPePi/+OBPGfb2bnmtlyM/vLzO5KsH89M5tlZv+E\n+z5rZrnDbV+Fuy0Jv2/3BOe/08zWA6/FtYXHnBxeo1b4+Xgz+8PMmqfr/1jJWnHPY8ukrsgkbAi7\nFwn/uzFs/w04IcF+ZcO238L3idv3Oyb8u1oY+PNQASixSVY5E8gLfJTMPv8DGgA1gDOAesDdCbaX\nJvhBL0MwyPycmRV19/uAh4FR7l7Q3V9JLhAzKwAMBdq6+zFAQ2DxQfYrBnwW7lscGAJ8ZmYJu0Yu\nAi4HSgG5gduSuXRpgj+DMgSJ+CXgEqA20AS4x8zKh/vuBW4GShD82Z0NXAvg7k3Dfc4Iv++oBOcv\nRlC97teF5O4rgTuBt8wsP/AaMMLdpyUTrxzxMneMLQljgcvC95cBYxK09wh/YS1PMElkbthtucXM\nGoTjZz0THRN3ri7A5+E4XLKU2CSrFCfo5kiue+xi4AF33+jufwD3A5cm2L4n3L7H3ccBW4G0jiHt\nA6qaWT53X+fu3x5kn3bAT+7+prvHuvu7wA9A+wT7vObuy919BzCaICknZQ8w0N33ACMJktbT7v5v\neP3vCBI67r4gHFyPdffVwItAsxR8p/vCwfodiTe6+0sEA/hzgOMIfpEQSTEzexeYBZwW9hD0BgYD\nrczsJ6Bl+JnwZ3o0wc/1BILemr3hqa4FXib4eVwJjA/bXwGKm9kKIG7o4pA0xiZZ5U+Cgenkxn6O\nB9Yk+LwmbIs/R6JjtwMFUxuIu28zs+4E1dUrZjYDuNXdfzhEPHExlUnweX0q4vkzwV/suMSzIcH2\nHXHHm9mpBBViHSA/wd/dBcl9L+APd995iH1eIvituK+77zrEvpIdZOKSWu5+YRKbzk5i/4HAwIO0\nzweqHqR9J5DqyV+q2CSrzAJ28d+03oP5naAbLc6JYVtabCNICHFKJ9zo7hPdvRVB5fIDwT/4h4on\nLqbfDrJvRnuBIK6K7l4IuItgRCU5yXbZmFlB4CmC34oHhF2tkt1lfVdklsu+kUu25u6bCcaVngsn\nTeQ3s1xm1tbMHg13exe428xKhpMw7gXeSuqch7AYaGpmJ4YTV/rHbTCzY82sYzjWtougS3PfQc4x\nDjg1vEUhZ1jlVQE+TWNMqXEMsAXYamaVgGsSbd8AVEjlOZ8G5rv7lQRjh8PSHaXIEUCJTbKMuz9B\n0G9+N/AHwf0q1wMfh7s8BMwHlgLfAAvDtrRcazIwKjzXAvZPRjFhHL8T3GzajAMTB+7+J3AecCtB\nV+odwHnuviktMaXSbQQTU/4lqCZHJdo+ABgRzprsdqiTmVlHgtUd4r7nLUCtuNmgko1l4pJaRypL\nwQQTERHJBmKKlvM8ze8+9I4ptPPjPgvScx9bVlHFJiIikaJZkSIiUZKNuxAziio2ERGJFFVsIiIR\nYqrYlNhi8hbyHAVLZnUYko1VP6nooXcSScbChQs2uXu6/yEylNhAiY0cBUtStOOgrA5DsrEZLx5y\ndr1IsvLlssQr2kg6HPWJTUQkMoxDr0dzFFBiExGJDFNXJJoVKSIiEaOKTUQkQlSxKbGJiESKEpu6\nIkVEJGJUsYmIRIgqNiU2EZHo0HR/QF2RIiISMarYREQiwnQfG6DEJiISKUps6ooUEZGIUcUmIhIh\nqthUsYmISMSoYhMRiRBVbEpsIiLRofvYAHVFiohIxKhiExGJEHVFKrGJiESGbtAOqCtSREQiRRWb\niEiEqGJTYhMRiRblNXVFiohItKhiExGJClNXJCixiYhEihKbuiJFRCRiVLGJiESIKjYlNhGRyNAN\n2gF1RYqISKSoYhMRiRIVbKrYREQkWlSxiYhEhe5jA5TYREQiRYlNXZEiIhIxqthERCJEFZsSm4hI\ntCivqStSRESiRRWbiEiEqCtSiU1EJDLMtKQWqCtSREQiRolNRNLsn3/+4cLuXTijaiVqVKvM7Fmz\nAFi6ZAnNGp9JnRrVuOD89mzZsgWAeXPnUr92DerXrkG9Wmcw5uOPsjL8SIqr2jLilV0psYlImt12\n8020bt2GJct+YO6CJVSqXBmAa666koceHsz8xd/QoWMnnnziMQBOr1qVGXPmM2fBYsZ8NoEbrr2K\n2NjYrPwKkaPEpsQmImm0efNmpk//il5X9AYgd+7cFClSBIAVPy2ncZOmALRo2YqPP/oAgPz585Mz\nZzC0v2vnzmz9j6ccuZTYRCRNVq9aRYkSJenb+3Ia1KnJNX2vZNu2bQBUrnI6n4wdA8CH77/Hr2vX\nxh83d84cap1xOnVqVmPoc8PiE51kEMvAVzalxCYiaRIbG8viRQvpc9U1zJ6/iPwFCvD4o4MBePGl\nVxk+7Hka1qvN1q3/kjt37vjj6tWvz8Il3zJ91jwee2QQO3fuzKqvIBGlxCYiaVKmbFnKlC1Lvfr1\nAeh0QRcWL1oIwGmVKvHp+EnMnLuAbt0vpHyFkw84vlLlyhQsWJBvly3L1LijTmNsSmwikkalS5em\nbNkTWP7jjwBM+3wqlSpXAWDjxo0A7Nu3j8EPP0SfvlcDQfdl3GSRNWvW8OOPP3BSuXKZH3xUmRIb\n6AZtEUmHIU89w+U9L2b37t2Uq1CB4S+/BsDoke/y4rDnAOh4fmd69rocgJkzpvP4Y4PJlTMXMTEx\nPP3M85QoUSLL4pdoMnfP6hiyVK4SJ3vRjoOyOgzJxn55sVtWhyDZXL5ctsDd66T3PHlLV/SylwzN\niJAAWPnEuRkSV2ZTxSYiEhnZuwsxo2iMTUREIkUVm4hIhKhgU8V21OnTsiJfPnAOXz1wDn1bVoxv\nL1IgN+/d0pTZD7flvVuaUjh/LgCKFsjNh7c3Z9VznRh0Uc39ztWp3glMu7810wa0ZuT/NaFYwdzI\nkee0U8pRp0a1+DUaZ82cmez+JYoUTPc1+1zRi0oVy1O/dg3OrFsrfg3J1Pj0k7E8Ft4XN3bMx3z/\n3Xfx2x4YcC+fT52S7jijSLMiVbEdVSqVKcQlTSvQ5qEp7I7dx6ibmzJ56TpWbdzKjW0r8dX3G3lm\n/A/c0LYSN55bmQffX8quPXt55KNlVCpTmEplCsWfK0eM8dCFNWlyzwT+2rqbe7tUp3eLijw29tss\n/IaSlAlTvsj02YcPD36Mzhd0YcrkSdxw7VXMW7Q0Vcef174D57XvAMAnYz6mbbvzqFwluJ3g3gEP\nZHi8Eh2q2I4iFY8rxMKf/2TH7r3s3efM/PEP2tUqA0CbmsczauZqAEbNXE3bmscDsH33Xuas2MTO\n2L37ncsseOXPE/xudEy+XKz/Z0fmfRlJl61bt9K29dmcWbcWdWpUi1/+KqF169bR8qym1K9dg9o1\nqjJ9+tcATJk8iWaNz+TMurW4qEdXtm7dmuy1GjdpysqVKwBYsngxTRs1oG7N6nTr0om///4bgOee\nGUrN6lWoW7M6l17cA4A3R7zO/914PbNmzuSzT8dyV7/bqV+7Bj+vXEmfK3rx4QfvM2niBC7q0TX+\nWl99OY3OHc9LU5yRYP/93cyIV3alxHYU+eG3zTSoWJKiBXKTL3cOWlYvzfHF8gNQslBeNm4Oljba\nuHknJQvlTfZcsXudO95cyJf3n8M3T7Tn1OML8fbXqw77d5C0adPyLOrXrkGThsEqIXnz5mXU+x8x\na95CJkz5gn533EriW39GjXyHVq3PYc6CxcxdsIQzzqjBpk2bGPzwQ4ybOIVZ8xZSq3Ydhj41JNlr\nf/bpJ5xetRoAV17ek4GDHmHeoqVUrVqNgQ/eD8Djjw1m9rxFzFu0lGeeG7bf8Wc2bEi78zrw8ODH\nmLNgMRVO/m8VkxZnt2Te3Dnxa1S+P3oUXbv1SFOcUWBATIxl2Cu7UlfkUeSndf/yzPgfGH1LU7bv\n3suyX/5h776D38d4qNsbc+Ywep11MmffP4nVf2xj0EU1ualdJZ789PvDELmkV+KuSHfn3rvvYsbX\nXxETE8Pvv/3Ghg0bKF26dPw+derU5ao+V7Bnzx7adzifM2rU4OuvvuSH77+jRdNGAOzes5v69c88\n6DXv6nc7jzz8ECVKlmTY8FfYvHkz/2z+hyZNmwFwyaWXcXFYbVWrVp1ePS+mQ4fzad/x/BR/r5w5\nc9K6dRs++/QTOl/QhfHjP2Pg4EdTFadEjxLbUead6at4Z3pQWd3VuRrr/t4OwB9bdlKqcFC1lSqc\nl03/Jr8wbdUTgseTrP4j+E15zPy13Ni28mGMXDLSyHfeZtOmP5g5dwG5cuXitFPKsSvRYsSNmzRl\n8udfMWHcZ/Tt3Ysb/+8WihQtSouWrXjjrXcPeY24MbY4mzdvTnLfj8Z+xvSvv+KzTz/hkcEDmb/o\nmxR/l67de/DC889SrFgxatWuwzHHHIO7pzjOqMnsLkQzuxm4EnDgG+ByID8wCigHrAa6ufvf4f79\ngd7AXuBGd58YttcGXgfyAeOAmzyNK4ioK/IoU+KYPACUKZafdrXK8MHsXwCYuPh3ujcsB0D3huWY\nsOj3ZM+z7p8dnHZcIYoXDM7XrEppflq35fAFLhlq8+bNlCxZily5cvHltC/4Zc2aA/ZZs2YNxx57\nLFdc2YdeV1zJokULqVe/AbNmzmDlimDMbNu2bfy0fHmKrlm4cGGKFikaP1b3zttv0rhpM/bt28ev\na9fSrPlZDBz0CJs3bz5gPKzgMcew9d9/D3reJk2bsXjRQl595SW6dgvG59ITZ3aXmbMizawMcCNQ\nx92rAjmAHkA/YKq7VwSmhp8xsyrh9tOBNsDzZpYjPN0LQB+gYvhqk9Y/A1VsR5lXr21I0YK5id3r\n9Ht7IVt27AFg6LgfeOmaM7m4SXl+/XM7Vw77b3r2/EfacUy+nOTOEUPbmmXoNuQrlq/bwuNjv2PM\nnWcRu3cfa//czo2vzs2qryWp1OOii7ng/PbUqVGNWrXrcFqlSgfs8/WX03hyyGPkypmLAgUL8spr\nb1CyZEleeuV1el5yIbt37QLgvgceouKpp6boui+9OoIbrruaHdu3x68tuXfvXi6/7BK2bN6M41x7\n/Y3xDyyN07VbD667pg/PPzuUd0a9v9+2HDly0Pbc83jrjdd5+dURAOmOU1IlJ5DPzPYQVGq/A/2B\n5uH2EcA04E6gIzDS3XcBq8xsBVDPzFYDhdx9NoCZvQGcD4xPS0BaK1JrRUo6aa1ISa+MWisy3/Gn\n+im9n8uIkABY9lDrQ8ZlZjcBA4EdwCR3v9jM/nH3IuF2A/529yJm9iww293fCre9QpC8VgOD3b1l\n2N4EuNPdz0tL3OqKFBGRpJQws/kJXn0TbjSzogRVWHngeKCAmV2ScJ9wnCxTKyh1RYqIRIRBRq8Y\nsukQFVtLYJW7/0Fw7Q+BhsAGMzvO3deZ2XHAxnD/34ATEhxfNmz7LXyfuD1NVLEdwcb/72w+v68V\nCx9tx3dPdeDz+1rx+X2tOKF4/gy9TvlSBdn4Sjd6Nf/v/qDHLq1NlwYnZuh1ihTIzWXN/rvG8UXz\nMfyqBhl6DTm0Jg3rU792DSpWOJETjisZv9TWmtWrD8v1Btx7N888/RQAl/e8hLFjPj5gn8t7XhK/\nBFf92jU4u3mTwxJL9GXcxJEUJshfgAZmlj/scjwb+B4YC1wW7nMZELcCwFigh5nlMbPyBJNE5rr7\nOmCLmTUIz9MzwTGpportCNZ24FQAujcqR42TitL/nUUH3S/GjH3pHCvduHkHV7U6lTe/+jnJe9vS\nq2iB3FzWvAIjvlwJwO9/76Dvi7MPy7UkaV/PnAMEK3ssWDCfp4Y+m8URBR59/Ek6JHMPW2xsLDlz\n5kzyc0qPk4zj7nPM7H1gIRALLAKGAwWB0WbWG1gDdAv3/9bMRgPfhftf5+5xyxpdy3/T/ceTxokj\noMSWLeWIMX54uiMjZ6ymSaVS3PbmAl6++kya3juRLTv2ULtCMfp3qkaXJ76kQJ6cDLq4JqceV4hc\nOWN45ONlTFqy7oBzbty8kyVr/qbrmScxcsbq/baVL1WQwRfXoljB3GzfvZebX5/Hzxu2Ur5UQZ7v\nU598uXMwcfHvXNHiFCre8DEF8+ZkxPWNKJw/NzljjIEffsPkpeu4+4JqnFz6mKDyXLaet79exSvX\nnEmL+ycz6Z6WXDN8Dis3BFO6P+l3Fv3fXsSqjVtTFL+k3ysvDeenn5Yz+NHHARg+7AVWrfqZK/tc\nRdcLOnJ61Wp8s3QJp1etxsuvjiBfvnzMnzeP/nfexratWylZqhTDX3mdY489NsNiGnDv3az95Rd+\n/nkl5cqVp2mz5owb9ylbNm8mJiaGzyZMpt8dtzF1yiQM46577qPzBV34fOoUBj/8EAULFuTnn1ey\n+JujZ+GAzL6Pzd3vA+5L1LyLoHo72P4DCSabJG6fD1TNiJjUFZlNFc6fm9nL/6D5gEnMX/lnkvvd\n2r4Kny9bT5uBU+n82DTu71aDPDkP/n/70HE/cF2b0w74i/HEZXW4860FtHpwCgM/WMrgi2sBMOii\nmjw/8Uea3zeJDZv/u7l35569XPbsDFo+MJkuT3zJgz1qAPDQB9+wcv2/tLh/Mg99sP8NuGPmrqVj\n3aDr/bii+ShSIDfL1v6Tqvglfbp278HYMR8RGxsLwBsjXuOyXlcA8P1333H9Df/H4m++J2+evLw8\n/EV27drFbbfcxLujP2Dm3AX0uOgSHrjvnjRf/47bbo7viuzdq2d8+48//sD4SVN57Y23AFiyeBEj\n3/uQ8ZOm8sH77/HjD98zd8ESPp0wmTtuu5mNG4PhnIUL5vPUM88fVUkNtLo/HMaKzcwcGOLut4af\nbwMKuvuAw3XNg8TwOvCpu79/qH2zm1179vLZwkOPrTY//VhaVCvNjW2D+5Ty5IqhTPH8/LzhwAVh\nV23cyrJf/uH8uv+N7RbKl4vaFYrx6rUN49ty5AgSS60Kxbnw6eBm2w/n/EL/TsEvW4ZxT5fq1D+l\nBPvcOb5Y/kM+0mbM/LW8dUNjhnz6HefXPYFP5v+a6vglfQoVKkTjxk2ZOGE85ctXIEeOHFSqXJmV\nK1ZQrnx56jcIxkMvvPgSXnl5OE2bNef7776l3TktAdi7dy9lypZN7hLJSqorsn2HjuTN+9/apS1b\ntqZo0aIAzJwxnW7dLyRHjhyULl2aho0as3DBfHLnzk39Bmdy4okZO04s2cPh7IrcBXQ2s0Huvim1\nB5tZTnePPQxxRcLOPfuvth+7z+MXLc2TK0d8u5nR69kZ8UtfHcpTn33PsL4NWBBWgWbw19bdtLh/\ncopj69bwJArly8XZD0xm7z5n8WPn7RfTwfz653a27Yrl1OMK0bHuCdz46rw0xS/p0+uKKxn69BBO\nOqkcPS+7PL498W/vZoa7U7VadaZO+/qwxpQ/f4H9PxcokMSeiY5L4X6Rks1X5c8oh7NPJ5ZgEPHm\nxBvMrJyZfW5mS81sqpmdGLa/bmbDzGwO8KiZDTCzEWb2tZmtMbPOZvaomX1jZhPMLFd43L1mNs/M\nlpnZcMvONXQard20jTNOCn6LPa/2f781f7FsPb3P/u+BolVPLHLAsQn9+PsWVm/cSotqwWK4m7fv\nYcM/Ozi3ZvB4GzM4vWxhABat+iv+sTed6u1f5f2xZRd79znNqhwb/wSBrTv3UDBvriSv/fG8tdzU\nrhJ5cuVgebg8V2rjl/Rp2KgRq1au5MMP3qNLt+7x7atXrWL+vOCXjVHvvkPDho2pXKUKv//+G/Pm\nBivO7N69m+++zdzn8TVq3IT3Ro9k3759bNiwgVkzZ1Crdrrvc8624qb7H+1dkYd7sOI54GIzK5yo\n/RlghLtXB94GhibYVhZo6O63hJ9PBloAHYC3gC/cvRrBXe7twn2edfe64Vpl+YA03a2enT029lse\nuaQWE+9uyZ7YffHtj4/9lvx5cjLt/tZ89cA53N7h9EOea8in31G2+H+/7V714mwua34yXwxozdcP\ntKHVGcGz2u56ZxHXt6nEtAGtObFEgfjlud6btYa6pxRn2v2tOb/eCaxcH0wI+WPLLpas+Ztp97fm\n7guqHXDdsfPX0rn+iYyZtzZd8Uv6dLqgC40bN6Vw4f/+2laqXJmhTw+hRrXKbN+xnd59+pInTx7e\nGfk+d95+C3VrVqdB3ZrMmzsnzddNOMZWv3YN9u7de8hjOl/QhVNPq0TdWtVpd05LHnlsCKVKlUpz\nDBINh21JLTPb6u4FzewBYA9BIiro7gPMbBNwnLvvCauude5eIhwT+8LdR4TnGADscfeBZhYTniOv\nu3t43r/c/SkzuwC4g2CdsmLAM+4+OKkxtvDu+b4AMQVK1C7ePeOWoDma5M+dg+27g398ujQ4kXNr\nleWK52dmcVSZL2pLanVo14bb7+wf/3iZlStWcFH3LsxZsDiLI4uujFpSq0CZ07zyNcMOvWMKLbin\nRYbEldkyY7r/UwT3OLyWwv0TD6bsAnD3fWa2J8FjDPYBOc0sL/A8werSa8NkmOxTMt19OEE3KblK\nnHx0L5aZDjXKF+OhHjWIMeOf7bu5KRwXk+zpzz//pFnjBtSqXSc+qUn2k527EDPKYU9s7v5XeENe\nb+DVsHkmwaML3gQuBtIz+hyXxDaZWUGgCxC5WZBHopk//pGqSSVyZCtevDjLvv/pgPaTTzlF1Zpk\nK5l1g/YTwPUJPt8AvGZmtwN/EDyYLk3c/R8zewlYBqwHVDaIyFFLBdthTGzuXjDB+w0E419xn9cQ\nTAhJfEyvRJ8HJHPOAQne3w3cfajziYhEmqkrErTyiIiIRIzWihQRiYjgPrasjiLrqWITEZFIUcUm\nIhIZ2XvFkIyixCYiEiHKa+qKFBGRiFHFJiISIeqKVGITEYkOPbYGUFekiIhEjCo2EZGIiHse29FO\niU1EJEKU2NQVKSIiEaOKTUQkQlSwKbGJiESKuiLVFSkiIhGjik1EJCp0Hxugik1ERCJGFZuISESY\nVvcHlNhERCJFeU1dkSIiEjGq2EREIiRGJZsSm4hIlCivqStSREQiRhWbiEhEmGnlEVBiExGJlBjl\nNXVFiohItKhiExGJEHVFKrGJiESK8pq6IkVEJGJUsYmIRIQRrBd5tFNiExGJEM2KVFekiIhEjCo2\nEZGoMD22BlSxiYhIxKhiExGJEBVsSmwiIpFh6LE1oK5IERGJGFVsIiIRooJNiU1EJFI0K1JdkSIi\nEjGq2EREIiJ40GhWR5H1lNhERCJEsyLVFSkiIhGjik1EJEJUryWT2MysUHIHuvuWjA9HRETSQ7Mi\nk6/YvgWc/X8BiPvswImHMS4REZE0STKxufsJmRmIiIikT7CkVlZHkfVSNHnEzHqY2V3h+7JmVvvw\nhiUiIqkWPrYmo17Z1SETm5k9C5wFXBo2bQeGHc6gRERE0iolsyIbunstM1sE4O5/mVnuwxyXiIik\nQTYutDJMSroi95hZDMGEEcysOLDvsEYlIiKSRimp2J4DPgBKmtn9QDfg/sMalYiIpEl2HhvLKIdM\nbO7+hpktAFqGTV3dfdnhDUtERFJLsyIDKV15JAewh6A7UstwiYjIESslsyL/B7wLHA+UBd4xs/6H\nOzAREUk9TfdPWcXWE6jp7tsBzGwgsAgYdDgDExGR1Mu+6SjjpKRbcR37J8CcYZuIiMgRJ7lFkJ8k\nGFP7C/jWzCaGn1sD8zInPBERSSkzPY8Nku+KjJv5+C3wWYL22YcvHBERSY/MzmtmVgR4GahKUPxc\nAfwIjAKLWP9kAAAgAElEQVTKAauBbu7+d7h/f6A3sBe40d0nhu21gdeBfMA44CZ397TElNwiyK+k\n5YQiInJUeRqY4O5dwlWp8gN3AVPdfbCZ9QP6AXeaWRWgB3A6wYTEKWZ2qrvvBV4A+gBzCBJbG2B8\nWgJKyazIk81spJktNbPlca+0XExERA6vzJwVaWaFgabAKwDuvtvd/wE6AiPC3UYA54fvOwIj3X2X\nu68CVgD1zOw4oJC7zw6rtDcSHJNqKZk88jrwGsFkm7bAaIISU0REjjBmGfcCSpjZ/ASvvokuVx74\nA3jNzBaZ2ctmVgA41t3jJhmuB44N35cB1iY4/tewrUz4PnF7mqQkseWP6wN195XufjdBghMRkWjb\n5O51EryGJ9qeE6gFvODuNYFtBN2O8cIKLE1jZWmVkvvYdoWLIK80s6uB34BjDm9YIiKSWoZl9qzI\nX4Ff3X1O+Pl9gsS2wcyOc/d1YTfjxnD7b0DCh1iXDdt+C98nbk+TlFRsNwMFgBuBRgSDe1ek9YIi\nIhIN7r4eWGtmp4VNZwPfAWOBy8K2y4Ax4fuxQA8zy2Nm5YGKwNyw23KLmTWwYHCvZ4JjUi0liyDH\nZeJ/+e9hoyIicqSxzJ/uD9wAvB3OiPwZuJygaBptZr2BNQRPhcHdvzWz0QTJLxa4LpwRCXAt/033\nH08aZ0RC8jdof0Qy/aLu3jmtFxURkcMjs9d4dPfFQJ2DbDo7if0HAgMP0j6f4F64dEuuYns2Iy5w\npKtctgifPtohq8OQbKxo3euzOgQRSSC5G7SnZmYgIiKSfnquWMqfxyYiIkc4Q0/QBiV3ERGJmBRX\nbGaWx913Hc5gREQkfWJUsKVorch6ZvYN8FP4+Qwze+awRyYiIqkWYxn3yq5S0hU5FDgP+BPA3ZcA\nZx3OoERERNIqJV2RMe6+JtGA5N6kdhYRkawRLF6cjUutDJKSxLbWzOoBbmY5CO4y12NrRESOQNm5\nCzGjpKQr8hrgFuBEYAPQIGwTERE54qRkrciNBE88FRGRI5x6IlOQ2MzsJQ6yZqS7J37gnIiIZCGD\nzH5szREpJWNsUxK8zwt0Yv8noIqIiBwxUtIVOSrhZzN7E5h+2CISEZE003JSafszKA8cm9GBiIiI\nZISUjLH9zX9jbDHAXwSP/hYRkSOMhtgOkdjCR3SfAfwWNu1z9yQfPioiIlnHzDR5hEN0RYZJbJy7\n7w1fSmoiInJES8kY22Izq3nYIxERkXQLltXKmFd2lWRXpJnldPdYoCYwz8xWAtsIbpVwd6+VSTGK\niEgKaUmt5MfY5gK1gA6ZFIuIiEi6JZfYDMDdV2ZSLCIikg5aeSSQXGIraWa3JLXR3YcchnhERCQd\nlNeST2w5gIKElZuIiEh2kFxiW+fuD2RaJCIikj6mySOQgjE2ERHJPkz/dCd7H9vZmRaFiIhIBkmy\nYnP3vzIzEBERSZ9gVmRWR5H1UvI8NhERySaU2PToHhERiRhVbCIiEWK6kU0Vm4iIRIsqNhGRiNDk\nkYASm4hIVGTzx81kFHVFiohIpKhiExGJEK3ur8QmIhIZGmMLqCtSREQiRRWbiEiEqCdSiU1EJEKM\nGK3ur65IERGJFlVsIiIRYagrElSxHXVuu/EqalU6kVaNa+/X/u03Szj/nKa0bV6f885uxOKF8wD4\netpU2rVoSOsmdWjXoiEzvpoGwI7t2+nVoxMtGpxBy0a1GPzA3Zn9VUQksfAJ2hn1yq6U2I4yXXtc\nyohRYw5oH3T//7jp9v8xftocbul3D4MG/A+AosWK8+rb7zPp6/kMee4lbr72ivhj+l73f3w+ewnj\nvpjN/Lmz+GLKxEz7HiIiSVFX5FGmfsPGrP1lzQHtZsbWf7cA8O+WzZQqfRwAVavXiN/n1EpV2Llz\nJ7t27SJf/vw0bNIMgNy5c1O1eg3W//5bJnwDEUmObtBWYpPQvQMfo2fX9gy8rz/79u3jw/FfHLDP\nuE8+omr1GuTJk2e/9s2b/2HKxHFc0ff6zApXRCRJ6ooUAN56bTj3PPQos5eu4N6HHuWOm67Zb/vy\nH75j8AN3M+iJZ/drj42N5Ya+l3F5n2s5sVz5zAxZRBKJmzySUa/sSolNAPhg5Nu0Pe98ANp1vIAl\nC+fHb1v3+6/07dmdIc+9zEnlK+x3XL9brqN8hZPpffUNmRqviBxcjFmGvbIrJTYBoFTp45g942sA\nZnw9jXIVTgGCbsbLL+zMnfc+SN36Dfc75rGHB/Dvls3cN/DxTI9XRCQpGmM7ytzQpyezZnzN339t\non61k7n5znvocUkvHnnyOQbcdTt798aSJ08eBg8JuhxHvDyM1atWMvTxQQx9fBAAb773CXv27ObZ\nIY9wcsXTaNfiTAB69r6aCy+9PMu+m4hk7y7EjKLEdpR55qU3Dtpet0EjPvt85gHtN97ajxtv7XfQ\nY9Zs2pGhsYlI+hjqhgP9GYiISMSoYhMRiQoL7kk92imxiYhEiNKauiKPSI1qnkbrJnVo27w+bZvX\nZ/7cWcnuX/mkEum+5q3X96Fe1Qrs2rULgL/+3ESjmqel+7yJTRw3luU/fh//+YlBDzD9y88z/DqS\ncYbddzFrpg5i/nt37dfeuWVNFrz/P7YtGEqtKifut+22K1qzbMx9LPnoHlqeWTm+fcB17flp/IP8\nMeOJ/fZvVOtkZr5zJ//Oe5pOLWsgkh5KbEeokR9PYPy0OYyfNoc69c7MlGvmyJGD0W+POKzXmDTu\nE35KkNhu7X8vjZu1OKzXlPR585PZdLzuuQPav135Oz1ufYnpC1fu116pQmm6nlOLWl0G0uG653m6\nfzdiwhV1x331DU0ufeyAc61d9zd973uTURPmH7BNUs7QfWygxJZtbNu6lQs7teXcs86kdZM6TBr3\nyQH7bFi/jq7ntaRt8/q0alybubOmA/DVF1M4v00zzj3rTK654iK2bd160GtccdX1vDLsGWJjYw/Y\nNuyZIbRv2YhzmtZlyOAH49uffnwQZ9WvzgXtWnBDn568+OyTALz7xqu0b9mINs3qcVWvHuzYvp35\nc2cxecJnPDzgLto2r8+aVT9z6/V9+Gzsh0ybOolrrrgo/ryzpn/F5Rd2TlX8cnjMWLiSvzZvP6D9\nx1Ub+GnNxgPaz2tenfcmLmT3nljW/P4nK9duom7VcgDM/WY16zdtOeCYX9b9xbKffmffPs/w+I82\nloGv7EqJ7QjV4/w2tG1en46tmwCQJ29ehr8xinFfzGLkxxN46L5+uO//j8CYD0bRtEUrxk+bw4Qv\n51Kl6hn89ecmnnliMO98MI5xX8yieo1avPzC0INe8/iyJ1CnQUM+HP3Ofu1ffTGF1T+vZOzk6Yyf\nNodvlixizszpLFk4nwmffsz4L+cyYuQYli5eGH9Mm/M68smUGUz4ci6nnFqJkW+/Tp16Z9KqTTvu\nGvAw46fN2W8Vk8bNWrB4wTy2b9sGwKcfv0/7Tl1TFb8cGcqULMyv6/+O//zbxr85vlThLIxIjjaa\nPHKEGvnxBIoV/2/szN159KF7mTtrBjExMaxf9zt/bNxAqWNLx+9zRs063H7TVcTu2UPrc9tzerUz\nmDrxa35a/gMXtAu6+3bv3k2tuvWTvO51N93OlZd2pUWrNvFtX30xha+nTeHcsxoAsG3bVlb9vIJt\nW/+lVZvzyJs3L+TNy9nnnBt/zI/ff8fjgwawZfNmtm3bSrOzWiX7fXPmzEmzFq2ZMvEzzu3Qmc8n\nj6f/fQOZMzN18Ysc7bJxD2KGUWLLJj5+fyR/bdrEp1NnkitXLhrVPC1+okec+g0b897YyXw+eQK3\n3dCXK6+5kcKFi9CkWYskb8xOrPzJp1ClanU+HfNBfJu7c+1Nt3Nxryv32/eVYc8keZ5bb+jDS2+M\npkrV6rz37pvMnvHVIa/dvlNXRrzyAkWKFqNajVoUPOYY3D1V8UvW++2PzZQtXTT+c5lSRfl94+Ys\njOhoYpruj7ois41/t2ymeMmS5MqVi5lff8mva385YJ9f166hRKljubDnFXS/pBfLli6iZp16zJ87\ni9U/BwP827dt4+cVPyV7rRtuuZOXnnsq/nOzFq0Y/c6I+LGt9et+Y9MfG6lT70ymTBzHzp072bZ1\nK59PGh9/zLatWyl1bGn27NnDx++PjG8vULBgkmNkDRo14duli3n3zVfp0KkrQJril6z12bSldD2n\nFrlz5eSk44tzyoklmbdsdVaHJUcRVWzZxPldenDFxRfQukkdqteoxckVD5yKP3vG17z47JPkzJWL\nAgUKMOS5VyheoiSPP/MSN/Ttye7duwG4rf99VDilYpLXOrVSFU6vXoNvly4GoOlZLVmx/Ac6tW0O\nQP4CBXj6hdc4o1YdWrVpR5umdSlRqhSnVTmdQoWCsZRb+91Lx3OaUrx4CWrUrhufzNp36kq/m6/j\ntZeeZ9ir+4/l5ciRgxat2/L+yLcY8uzLAGmKXzLWiEG9aFK7IiWKFGTFhAd5cNg4Rnw8iw5nVWfI\nnV0pUbQgHw69mqU//kaH657j+5/X88GkRSz64H/E7t3H/w0eHT8pZOBNHenetg758+ZixYQHee2j\nWQx8cRy1q5zIqCF9KFIoP+c2rcbdV7ejdpeBWfzNsx8tqRWwxBMQjjbVa9T2T6fOyOowsq1tW7dS\noGBBdmzfTtf2rRg05FmqnVEzq8PKVKe1vC2rQ5Bsbufi5xa4e530nufkKmf4w2+Py4iQAOhRq2yG\nxJXZVLFJuvS75TpWLP+BXTt3ckGPS466pCZypNEYmxKbpNMzww/vDd0ikjpKa0ps2VLH1k3YvXs3\n//z9Fzt37qT0cccDMPyN0Zxw4kkZfr3HHh5AsWLFD3hK9mMPD+C9d96geImS8W3vfTKFgscck+Ex\nSPp99cZt5M6dk2KF8pM3b674mYrdbh7OL+v+yrDrVDihBPNH38XyNRvJnSsHX877iZsHj071ecY+\ndx0X3f4yuXLm4ILWtXj5/WDBgbLHFmHQzZ24tN9rGRazRIsSWzY0ZlLwpOv33n2TpYsX8OAjTx3i\niMPnqutvPiDhJRQbG0vOnDmT/JwUd8fdiYnRUHhGadozeNL5Je3rU7vKidz8yHsH3S8mxtK9Asjy\nNRtp0GMwOXPGMOmlm2jXrBqffflNqs7RIVzGq8IJJbiyS+P4xPbrhn+U1JKi1f0BTaCJlHdGvMJD\n9/73UNA3Xx3OwPv6s/rnlbRsVIvrr7yUFmfW4NreF7NzR/CQ0CUL59OtfSvatWhIz+4d+WPjhnTH\n8e6br9Hn0q5073gOl3Ztz/QvP6d7h9ZcfmFnWjcJxqGHDX2CVo1r06pxbV576XkAVv+8krMb1uTG\nq3rRslEtNm5Yl+5Y5NBy5Ihh3VeP8thtFzB3VH/qVi3HigkPUrhgPgDqVSvHZ8OuB6BAvtwMv/8S\nvn7zNma9eyfnNq2a7LljY/cxZ+lqTj6hJGbGI7d2Zv57dzFv9F3xix0fX7IwU1+9mdkj+zH/vbto\ncEZ5gPgYHrqxI6eeVIrZI/vx4I0dqHBCCWaPDH7Op799BxVPKhV/vamv3kz1U8ukOs6oiJsVmVGv\n7Co7xy6JtO/clYnjxsav9fjeu2/Q7eLLAPjpx++54urr+XzWYvLkycvbI15m165dDPjfbQx7/V0+\n+3wmnbr04IlBD6Tqmi8++2T8Uwgu6vzfyiPffrOE4SNG8u5Hwb1tS5cs5KHHnuLzWYtZtGAuH38w\nik8mT+ej8dN489Xh/PDdMgBW/vQjV159A1NnLqL0cWUy4o9FUqDIMfmZvnAF9boPYs7SVUnud1ff\ntkye+T1NLn2ctn2HMviWzuTJnXQFnj9vbprVPZVlP/3OBa1qclr5Y6nXfRDnXfMMj956ASWLFuTC\ndnUZ99U3NOgxmHrdB/HN8t/2O8fdQ8fEV4D3DB2737YPJi7ggta1AChTqghFC+dn6fLfUh2npI+Z\n5TCzRWb2afi5mJlNNrOfwv8WTbBvfzNbYWY/mtk5Cdprm9k34bahlo7SM9P/nzaz84GPgMru/oOZ\nlQMauvs74fYawPHunqY5q2a2Gqjj7psyJuLs45hjClH/zMZMmzKRE8uVJyZHDiqeWonVP6/khJPK\nUatOsBRVp64X8s4br9CgUVN++uF7Lr6gHQB79+6l9PGpSyZJdUU2ad6SwkX+W32iVp36lCkbPNpk\n3uyZtD3vfPLmCyqCc85tz9zZM2javCUnlatA9Zq10/T9Je127d7DmM+XHHK/s8+sTOtGp3Pr5cES\naXlz5+SE0sVY8cv+iyHHVVj79jljv1jC53N+YMidXRk9YQH79jkb/vyXmYtXUuv0E5n/7S88e3cP\n8uTOxSfTlh6Q2JLzweSFvP/U1Qx+aQJdzqnFh5MXpSrOKMqirsibgO+BQuHnfsBUdx9sZv3Cz3ea\nWRWgB3A6cDwwxcxOdfe9wAtAH2AOMA5oA4wnDbLiV5gLgenhf+8DygEXAXF369YA6hB8MUml7pf0\n4uUXhlL2hJPodmHP+PbEP+xmhrtT6fSqvP/p1AyPI3+B/Pt/zp8/iT0TH1cgw2ORQ9uxa89+n2P3\n7ot/1Eye3Lni282g2y3DWfVr8r83xlVYKfHlvOWcc+XTtGlSlZcfvJQnX5/CyPEpe3zNL+v+ZtuO\nXVSqUJourWvR5763UhVnFGV2WjOzskA7YCBwS9jcEWgevh8BTAPuDNtHuvsuYJWZrQDqhQVJIXef\nHZ7zDeB80pjYMrUr0swKAo2B3gRZG2Aw0MTMFpvZncADQPfwc3czq2dms8Iyd6aZnRaeK4eZPW5m\ny8xsqZndkOha+cxsvJn1ycSvmOXq1m/IL6tXMW7sh5zXqUt8+9o1q1myMPjHYswHo6hbvyEVT6vM\n+nW/s3jhPCBYYHj5D98d9hjrndmICePGsnPHDrZt3cqk8Z9Sr0Gjw35dSbk1v/9FzcpBhZ3wwZ9T\nZn7PtT2axX8+47SyKT7njIUr6HpObcyMUsWO4cwzKrDw21848biirP9zC69+OIM3x8zmjEon7Hfc\n1m27OCZ/niTP+/7Ehdx+eWty587JDz+vT3ecsp8SZjY/wavvQfZ5CrgD2Jeg7Vh3jxskXw8cG74v\nA6xNsN+vYVuZ8H3i9jTJ7IqtIzDB3Zeb2Z9mVpugRL3N3c8DMLMNBF2J14efCwFN3D3WzFoCDwMX\nAH0Jqr0a4bZiCa5TEBgJvOHuR93quW3bd2LlTz/GL28FcMqplXj5haF8u2wplaqczkU9e5MnTx5e\nePUdBtx1K1v//Ze9e/fS59qbOLVSlRRf68Vnn+T9kW/Ff37l7fcPeUyNWnXp2Kkr7Vs1BuCSy/tQ\nqUrV+PUgJes9NGwcz997IZv/3cH0hSvi2we+OJ7Hbr+AeaPvIibGWLn2D7rdPDxF5/xwymLqVS/P\nvNH9cYc7h3zIH39vpWfHBtx4SQv2xO5l6/Zd9L57/3sjN/71L4u+X8u80XcxYfoyXvtoZqLzLuKR\nWzvzwAufZUic2V0G90RuSm7lETM7D9jo7gvMrPnB9nF3N7NMXeIqU5fUCgcWn3b3yWZ2I3Ai8Cn7\nJ7Ze7J/YTgCGAhUBB3K5eyUz+wAY5u6TE11jNbAZeNTd304ijr4EiZEyZU+oPXPx8gz/rlmpZ7cO\nXHvT7TRoFDzLbfXPK7nmiosYP21OFkcWTVpSS9Iro5bUqnj6GT5k5KSMCAmADtVLJxuXmQ0CLgVi\ngbwEY2wfAnWB5u6+zsyOA6a5+2lm1h/A3QeFx08EBgCrgS/cvVLYfmF4/FVpiTvTuiLDiqoF8HKY\nfG4HunHoLuEHCb5wVaA9wR/eocwA2iQ1q8bdh7t7HXevU6x4yYPtki39/defNKtblcKFi8QnNRGR\nw8Xd+7t7WXcvRzC89Lm7XwKMBS4Ld7sMGBO+Hwv0MLM8ZlaeoGCZG3ZbbjGzBuG/2z0THJNqmdkV\n2QV4M2EGNrMvCfplEy5V8W+iz4WBuGlSvRK0TwauMrMv4roi3T1u+YR7w9dzwLUZ+i2OYEWLFefL\necsOaC9X4WRVayJHiSPk/uzBwGgz6w2sIShicPdvzWw08B1BlXddOCMSgn+rXwfyEUwaSdPEEcjc\nySMXEkzzT+gDgiy/18yWmNnNwBdAlbjJI8CjwCAzW8T+ifhl4BdgqZktIZhZmdBNQD4ze/QwfBcR\nkSOQZej/UsPdp8UNKbn7n+5+trtXdPeWCYoO3H2gu5/s7qe5+/gE7fPdvWq47XpPxzhZplVs7n7W\nQdqGJrF73USfT03w/u7w2FiCqaW3JNwxLInjXJ7qQEVEJFvTrfgiIhFyhHRFZiktqSUiIpGiik1E\nJCKCRZBVsimxiYhEhakrEtQVKSIiEaOKTUQkQlSxKbGJiERKau8/iyJ1RYqISKSoYhMRiQgDYlSw\nKbGJiESJuiLVFSkiIhGjik1EJEI0K1KJTUQkUtQVqa5IERGJGFVsIiIRoVmRAVVsIiISKarYREQi\nI/VPvo4iJTYRkajQ6v6AuiJFRCRiVLGJiESICjYlNhGRyAhmRSq1qStSREQiRRWbiEiEqF5TYhMR\niRZlNnVFiohItKhiExGJEN2grcQmIhIpmhSprkgREYkYVWwiIhGigk2JTUQkWpTZ1BUpIiLRoopN\nRCQiDM2KBFVsIiISMarYRESiQs9jA5TYREQiRXlNXZEiIhIxqthERKJEJZsSm4hIdJhmRaKuSBER\niRhVbCIiEaJZkUpsIiKRYWiIDdQVKSIiEaOKTUQkSlSyKbGJiESJZkWqK1JERCJGFZuISIRoVqQS\nm4hIpCivqStSREQiRhWbiEhU6EY2QBWbiIhEjCo2EZEI0XR/JTYRkcgwNCsS1BUpIiIRo4pNRCRC\nVLApsYmIRIsym7oiRUQkWlSxiYhEiGZFKrGJiESKZkWqK1JERCJGFZuISISoYFNiExGJFmU2dUWK\niEi0qGITEYmIYHF/lWyq2EREJFJUsYmIRIVpuj8osYmIRIrymroiRUQkYo76iu2bJQs3nVQi35qs\njuMIVwLYlNVBSLamn6HknZRhZ8rEks3MTgDeAI4FHBju7k+bWTFgFFAOWA10c/e/w2P6A72BvcCN\n7j4xbK8NvA7kA8YBN7m7pyWuoz6xuXvJrI7hSGdm8929TlbHIdmXfoYyi2X2rMhY4FZ3X2hmxwAL\nzGwy0AuY6u6Dzawf0A+408yqAD2A04HjgSlmdqq77wVeAPoAcwgSWxtgfFqCUlekiIikibuvc/eF\n4ft/ge+BMkBHYES42wjg/PB9R2Cku+9y91XACqCemR0HFHL32WGV9kaCY1LtqK/YRESiJINnRZYw\ns/kJPg939+EHv66VA2oSVFzHuvu6cNN6gq5KCJLe7ASH/Rq27QnfJ25PEyU2SYmD/iCLpIJ+hjKB\nkeFDbJtS0oVsZgWBD4D/c/ctliC7urubWZrGytJKXZFySEn9hiaSUvoZii4zy0WQ1N529w/D5g1h\n9yLhfzeG7b8BJyQ4vGzY9lv4PnF7miixiYhEiWXg61CXCkqzV4Dv3X1Igk1jgcvC95cBYxK09zCz\nPGZWHqgIzA27LbeYWYPwnD0THJNq6ooUEYmQTJ4V2Qi4FPjGzBaHbXcBg4HRZtYbWAN0A3D3b81s\nNPAdwYzK68IZkQDX8t90//GkcUYkKLGJiEgauft0kq7tzk7imIHAwIO0zweqZkRcSmySLmZWGTgO\n+Nrd92R1PJJ9mJml9QZcSZrWilRik/TrQTAYvNfMZiq5SUrFJTUzawCsdvf1WRxSJCivafKIpN/9\nBEvmdAcahzOkRJJkZjXNLHf4/mSCbqnYrI1KokSJTVLNEtyk4u77CP5hWoeSm6TMAOCTMLmtAjYD\nuwHMLMbMcmRhbNlb+NiajHplV0pskioJx0XMrLWZNQeKAA8BvxAkt4ZKbpKYmcUAuHtH4G9gNFCQ\noOLPH27bB+TOohAlIjTGJqmSIKndAnQimLbbB3jZ3R82szuBvgQrd0/PskDliBL+QrQvfF/S3XuY\n2RhgFsHPynFmthfIBawzs/7uviMLQ87GsnGplUGU2CTVzKwlcJa7NzGzQUA94EIzw90fMbObCRY3\nFQH2+4XoRqCOmV3j7h3NbBjBtPBHgRwE1f+PSmppY2TvLsSMosQmh3SQadlrgRvMrBdQFzgXeBIY\nYGa53P3JLAhTjnBm1olgFYrz3H0bgLtfbWbvAQ8C57u7JpFIummMTZKVaEytvpkVBVa5+2qC5XBe\nCJfDWQosARYneTI52lUAxrr7OjPLFTcO6+5dgQ0Ez+eSdMrEFbWOWKrYJFkJktrVwO3At8AkMxsJ\nLANGmFktoDPBb+IbkzyZHDWSuPn6N6CJmRVy9y3hft2AX929d6YHGVHqilRikyQkqtRKAdUJxtLq\nAK0IHu3+LMFU7fpAZ3dfmUXhyhEk0c9OZ+BfYCswCbgYuMLMfiQYT/sf0D6rYpVoUmKTAyT6h+l6\noDRwurv/CUwMp223BO4Annb3cVkXrRxpEk0UuYjgWWx3ECxy2xe4nuCXpLzAheGTlCWDZPIiyEck\njbHJARL9tn0ZMBcoa2ajwu3jga8Ipmbrb5EcwMxqAh2B5gTP1toIvAzUd/f/uftFQE93/ybroowo\nDbIpscl/Eq4oYma1CbqNhrv7WOAU4FQzexfA3ccAA8MqTo5yZlYkXB4LM6sO7AAuJEhurdy9KfAS\nMMrMLgFw961ZFa9Em7oiBTig+7ELUJlgdYjmZjbX3ZeEk0R+NrPX3b1X3JRtObqZWc7/b+/+Y72u\n6jiOP18SFr+CJgu3VuEvNNRgIAVajRlDLGH8EU1Fi2QotLm0olhaq60tm6uVwx9lP6xVZC11mjOG\nttQICCIhK35UrLJMoRJTNApf/XHOdZc70HvlG1++n+/rwb7j3u/3c8859+7uvr/nfM55v4FxwHm1\nWvJoYL7tPXUX7Xfqpf8APg+sbc9Iu0MHT7RaJoEtgP2WH2dR7oWcQwluFwFzJD1Xl42Oq5VvI3re\nEFI4ylUAAAaWSURBVP23bgb5GDAN+IjtPfWSlwHnSDqZsklkuu0/t2m4jdfpOR5bJUuR8bya93EJ\nsN72f2xvppRnHwZcKOlUgNzsD4A6G5tVPx1Hyfl4PTBJ0mwA28uB2yhnHOckqMXhkBlbFzvAWaMd\nlCz9x0uaYHuT7dX1IO3ZlEO0ET0GA2dJ+gSA7WmSRlN2Qs6W9AQlTdZeYEVPrsj4/8quyAS2rtXn\nntpsSj2sJ4DLgS8C83qWH23/RNK65O8LAEnH2v6b7cclPQaMp8zKsL1L0l2U36ePAhOAtyeoHUaJ\na1mK7HaS3k8pFvoW4GvAlfUxClggaTxAgloASDoF+KukL0i6ELiJsvNxp6Qb6humHcAq4BJgqu1t\nbRxydKEEti4j6XWShtl2zSjybsoOtquAM4HFwDxK8dBBlPNHET2eAn5GWbJeCNwIjARWAk8CyyVd\nTHlz9KTtv7RroN0qx9gS2LqKpDHAh4AlkobXvI67qNWLbf8TuAI4vSY2Xmp7V9sGHEcc249QDuxP\nouycvQ+4mJKd/y7gGGABsNz2s20aZnS5BLbushNYT8mi/r56IPt3wHfrWSSA11OyjAyi3CeJAPY7\nwL8MMOW82qPAZOBXlPuzjwDvtf2btgwynt/y34pHp8rmkS4g6STgKNtbJX2bkrj4XGCR7WWSbgQe\nkLSZktB4vu19bRxyHIHq8nXPn7vtwOcoQe1K23fU+2+P1Zl/tIWyK5IEtsaTdAywFdgl6VPAPkpS\n2pHAiZIus71E0pspSWk/m3NqcTB1J+1eSd8C7geut31HfW1LWwcXUSWwNZztv0uaAdxLWXqeANxK\n2QSwFzi9vgv/uu1/t2+k0Unq7H8ZMFbS0F6ZRqKNRGcvIbZKAlsXsP1jSecA11EC2xjKgevzKeVD\nTgZWAAlsMRBrKQVmI44oCWxdwvYqSR+mVL2eavsbku6kZI8Yant3e0cYncb2FknnZ7YWR5oEti5i\n+25JzwFrJU1LyZk4VAlqR54sRSawdR3b90g6GrhX0uSkOopoluyKzDm2rlSLhL41QS0imigzti6V\n6sURDdThB6tbJYEtIqIhOj3HY6tkKTIiIholM7aIiCbJlC0ztug8kvZJekjSw5K+L2noIbQ1XdIP\n68dzajaNg107qtavG2gfn6xnCPv1fJ9rbpH0rgH0NVbSwwMdY0STJLBFJ3rG9kTbp1HSgi3u/aKK\nAf9u277T9jUvcMkoYMCBLeJwUgv/daoEtuh0D1KSOY+VtFXSNynZVV4raaakNZI21pndcABJsyRt\nkbSRXimhJC2QtLx+PEbS7ZI21ceZwDXACXW2eG29bqmk9ZI21yTTPW1dJWmbpJ9SUpa9IEmLajub\nJP2gzyx0hqQNtb3z6vWDJF3bq+/LDvUHGc2QsjUJbNHBag25cym1wABOAm6wfSrwNHA1MMP2JGAD\n8EFJrwBuBmZTSq4ce5DmrwPutz2BUlTz15Q6ZL+vs8WlkmbWPt8ETAQmS3qbpMmUPJwTgXcAU/rx\n7dxme0rt77eU6tQ9xtY+3gncVL+HhcBu21Nq+4skHdePfiIaL5tHohMNkfRQ/fhB4KuU4ql/tL22\nPj8VGA+sriXEjgbWAKcAO2xvB6jlVy49QB9nA+8BqLXpdkt6VZ9rZtbHL+vnwymBbgRwe0+6qZqT\n88WcJunTlOXO4cDKXq99rx6m3y7pD/V7mAm8sdf9t5G172396CsarIMnWi2TwBad6BnbE3s/UYPX\n072fAlbZvqDPdft93SES8BnbX+rTxxUvoa1bgLm2N0laAEzv9Zr7XOva9+W2ewdAJI19CX1HkySy\nZSkyGmstcJakEwEkDZM0DthCqSF2Qr3ugoN8/X3Akvq1gySNBP5FmY31WAlc0uve3WskvRp4AJgr\naYikEZRlzxczAnhU0mBgfp/X5kk6qo75eErh2JXAkno9ksZJGtaPfiIaLzO2aCTbO+vMZ4Wkl9en\nr7a9TdKlwN2S9lCWMkccoIkPAF+WtJBSdXyJ7TWSVtft9PfU+2xvANbUGeNTwEW2N0q6FdgEPA6s\n78eQPw6sA3bW/3uP6U/Az4FXAottPyvpK5R7bxtrodidwNz+/XSiyTp5N2OrqFR6j4iITjd58hle\nvW5Dy9obMli/sH1Gyxo8TBLYIiIaQtKPgNEtbHKX7VktbO+wSGCLiIhGyeaRiIholAS2iIholAS2\niIholAS2iIholAS2iIholAS2iIholAS2iIholAS2iIholAS2iIholP8BOoGGwCUXTCUAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa098085b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(actual_value = df.loc[:,'Actual'].values.astype(int),\n",
    "     pred_value = df.loc[:,'Prediction'].values.astype(int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:47:53.425979Z",
     "start_time": "2017-07-23T21:47:53.418162Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Actual\n",
       "0.0    2152\n",
       "1.0    9698\n",
       "Name: Actual, dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_.groupby(by=\"Actual\").Actual.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:47:55.672387Z",
     "start_time": "2017-07-23T21:47:55.374976Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbQAAAGhCAYAAAAJL0FuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xm8TWX7x/HPdRyzEhmSKEpkyDxmSGQoU6ZIpbl+zVIa\nn2al4emR0qAJTQhFZE7JPESkVCSVpDTIUHK4fn/s5dimcw72Ocde+/t+vfbLWveari2d61z3ute9\nzN0RERGJd0nZHYCIiEgsKKGJiEgoKKGJiEgoKKGJiEgoKKGJiEgoKKGJiEgoKKGJiEgoKKGJiEgo\nKKGJiEgoJGd3ACIiknlyHH2ie8rfMTmX//3rJHdvFZOTZQIlNBGREPOUv8ldvmtMzvXPkoFFYnKi\nTKKEJiISagaWGHeXEuNbiohI6KlCExEJMwPMsjuKLKGEJiISdupyFBERiR+q0EREwk5djiIiEv80\nylFERCSuqEITEQk7dTmKiEjcM9TlKCIiEk9UoYmIhJqpy1FEREJCXY4iIiLxQxWaiEjYqctRRETi\nnx6sFhERiSuq0EREwiyBXh+jCk1EREJBFZqISNglyD00JTQRkVDToBAREZG4ogpNRCTskhJjUIgS\nmohImGm2fRERkfiiCk1EJOwS5Dk0JTQRkVDTKEcREZG4ogpNRCTs1OUoIiKhoC5HERGR+KEKTUQk\nzMwSpstRFZqIiISCKjQRkbDTPTSR+Gdmec3sfTPbaGbvHMZ5epjZ5FjGll3MrJGZfZXdcUgW2tXt\neLifI5wSmhwRzOwCM1toZpvNbJ2ZTTCzhjE4dWegOHCsu3c51JO4+5vu3iIG8WQqM3MzOyWtfdz9\nE3cvn1UxiWQVdTlKtjOzW4A7gGuAScC/QEugHTDzME9/IvC1u6cc5nlCwcyS9XeRaDRTiEiWMLOC\nwIPAde4+2t23uPt2dx/n7n2CfXKbWX8z+yn49Dez3MG2M83sRzPrbWa/BNXdpcG2B4B7gfODyu9y\nM7vfzN6Iuv5JQVWTHKxfYmbfmtkmM1ttZj2i2mdGHdfAzBYEXZkLzKxB1LaPzOwhM5sVnGeymRU5\nwPffFX+fqPg7mNk5Zva1mf1uZndF7V/HzOaY2Z/Bvs+aWa5g24xgt8+C73t+1PlvN7Ofgdd2tQXH\nnBxco0awfryZ/WpmZx7Wf1g5sqjLUSRL1AfyAO+msc/dQD2gGlAVqAPcE7X9OKAgUBK4HBhoZoXc\n/T7gEWC4uxdw91fSCsTM8gMDgNbufhTQAFiyn/0KA+ODfY8FngLGm9mxUbtdAFwKFANyAbemcenj\niPwdlCSSgF8CLgRqAo2A/5hZmWDfHUAvoAiRv7tmwLUA7t442Kdq8H2HR52/MJFq9aroC7v7KuB2\n4A0zywe8Bgxx94/SiFfkiKSEJtntWGBDOt1gPYAH3f0Xd/8VeAC4KGr79mD7dnf/ANgMHOo9op1A\nZTPL6+7r3H35fvY5F/jG3V939xR3fxtYAbSN2uc1d//a3f8GRhBJxgeyHejr7tuBYUSS1dPuvim4\n/hdEEjnuvsjd5wbX/Q54EWiSge90n7tvC+LZg7u/BKwE5gEliPwCIWGx631osfgc4Y78CCXsfgOK\n7OryO4DjgTVR62uCttRz7JUQtwIFDjYQd98CnE/kXt46MxtvZhUyEM+umEpGrf98EPH85u47guVd\nCWd91Pa/dx1vZqea2Tgz+9nM/iJSge63OzPKr+7+Tzr7vARUBp5x923p7CtxxZTQRLLIHGAb0CGN\nfX4i0l22S+mg7VBsAfJFrR8XvdHdJ7n72UQqlRVEftCnF8+umNYeYkwH43kicZVz96OBu4j8Dp4W\nT2ujmRUA+gOvAPcHXaoicUcJTbKVu28kct9oYDAYIp+Z5TSz1mb2eLDb28A9ZlY0GFxxL/DGgc6Z\njiVAYzMrHQxIuXPXBjMrbmbtg3tp24h0Xe7czzk+AE4NHjVINrPzgYrAuEOM6WAcBfwFbA6qx//b\na/t6oOxBnvNpYKG7X0Hk3uALhx2lHFmyaFCImZU3syVRn7/M7GYzK2xmU8zsm+DPQlHH3GlmK83s\nKzNrGdVe08yWBdsGmKUfgBKaZDt3/y9wC5GBHr8CPwDXA+8FuzwMLASWAsuAT4O2Q7nWFGB4cK5F\n7JmEkoI4fgJ+J3Jvau+Egbv/BrQBehPpMu0DtHH3DYcS00G6lciAk01Eqsfhe22/HxgSjILsmt7J\nzKw90Ird3/MWoMau0Z0SElnU5ejuX7l7NXevRmRQ01YiA77uAKa5ezlgWrCOmVUEugGViPw7fM7M\ncgSnex64EigXfFql+zXd0+yNEBGROJZ0zImeu8ld6e+YAf+MvWaRu9fKyL5m1oLIYKQzLDIzzZnu\nvs7MSgAfuXt5M7sTwN0fDY6ZROSXsu+A6e5eIWjvHhx/dVrX1IPVIiJhF7tnyIqY2cKo9UHuPugA\n+3YjcrsAoLi7rwuWfyYyew9EBlLNjTrmx6Bte7C8d3ualNBERMLMYjpTyIaMVGjBw/7tiLpHvYu7\nu5llSteg7qGJiEistQY+dfddj5+sD7oaCf78JWhfC5SKOu6EoG1tsLx3e5qU0EREwi7rp77qzu7u\nRoCxQM9guScwJqq9m0WmtytDZPDH/KB78i8zqxeMbrw46pgDSvgux0KFi3jJUqWzOwwJgdzJ+v1Q\nYuPTTxdtcPei2R3HoQgeezkbiB7A0Q8YYWaXE5mEoCuAuy83sxFEZsNJITKn665JBq4FBgN5gQnB\nJ00Jn9BKlirN6EmHO6G7CJQuki/9nUQyIG9O23smmsOSgUe4YiaYcefYvdp+IzLv6P727wv03U/7\nQiKz12RYwic0EZEwM7I2oWUn9ZGIiEgoqEITEQkzI/3ZPkNCCU1EJNRMXY4iIiLxRBWaiEjIJUqF\npoQmIhJyiZLQ1OUoIiKhoApNRCTkEqVCU0ITEQmzBBq2ry5HEREJBVVoIiIhZgn0HJoSmohIyCVK\nQlOXo4iIhIIqNBGRkFOFJiIiEkdUoYmIhFyiVGhKaCIiYabn0EREROKLKjQRkZBTl6OIiMS9RHqw\nWl2OIiISCqrQRERCLlEqNCU0EZGwS4x8pi5HEREJB1VoIiJhZupyFBGRkEiUhKYuRxERCQVVaCIi\nIZcoFZoSmohIiOnBahERkTijCk1EJOwSo0BThSYiIuGgCk1EJMz0HJqIiIRFoiQ0dTmKiEgoqEIT\nEQm5RKnQlNBERMIuMfKZuhxFRCQcVKGJiIScuhxFRCTumWnqKxGRg1L+lJOoVa0KdWtW44y6tVLb\n77z9NqpWrkDt6qfTtfN5/PnnnwC8/dab1K1ZLfWTL1cSny1Zkl3hSwgooYlIzEycOp15i5Ywa97C\n1LZmzc9m0ZLPWbB4KeXKncoTjz0KQPcLejBv0RLmLVrCK4Nf56QyZaharVp2hR5qu6q0w/0c6ZTQ\nRCRTNT+7BcnJkbsbderWY+2PP+6zz4jhb9Ola7esDi1hKKGJiBwEM+Pcls1pUKcmr7w0aL/7DB38\nKi1btd6nfeQ7w+l6fvfMDlFCTglNRGJi2kczmbdoCe+Nm8CLzw9k5icz9tj+2KN9yZGcTLcLeuzR\nPn/ePPLlzUelypWzMtzEYjH6ZORSZseY2UgzW2FmX5pZfTMrbGZTzOyb4M9CUfvfaWYrzewrM2sZ\n1V7TzJYF2wZYBkpEJTQRiYmSJUsCUKxYMdp1OI8FC+anbnt9yGA+GD+OwUPf3Kfr6p0Rw+jaTdVZ\niDwNTHT3CkBV4EvgDmCau5cDpgXrmFlFoBtQCWgFPGdmOYLzPA9cCZQLPq3Su7ASmogcti1btrBp\n06bU5alTJlOpUqTimjxpIk/993FGvjuWfPny7XHczp07GTVyhO6fZbKsuodmZgWBxsArAO7+r7v/\nCbQHhgS7DQE6BMvtgWHuvs3dVwMrgTpmVgI42t3nursDQ6OOOSA9hyYih+2X9es5v/N5AKTsSOH8\nbhfQomXkF+peN13Ptm3baNPqbCAyMOSZ514AYOYnMzjhhFKUKVs2ewJPBLF9fUwRM1sYtT7I3aNv\nmJYBfgVeM7OqwCLgJqC4u68L9vkZKB4slwTmRh3/Y9C2PVjeuz1NSmgictjKlC3L/E8/2++25StW\nHvC4xk3OZMasuQfcLkecDe5eK43tyUAN4AZ3n2dmTxN0L+7i7m5mnhnBqctRRCTEDDCLzScDfgR+\ndPd5wfpIIglufdCNSPDnL8H2tUCpqONPCNrWBst7t6dJCU1EJNRic/8sI92W7v4z8IOZlQ+amgFf\nAGOBnkFbT2BMsDwW6GZmuc2sDJHBH/OD7sm/zKxeMLrx4qhjDkhdjiIiEks3AG+aWS7gW+BSIsXT\nCDO7HFgDdAVw9+VmNoJI0ksBrnP3HcF5rgUGA3mBCcEnTarQEtCdva6hfuUTaXPmvl3hr7/yPK0a\nVufcJrV4/KG7AZj18TQ6tjiDtk1r07HFGcyZ+VHq/hd1bEXLhtVo37we7ZvX47cNv+xzTjlyRM+3\nWLdmNebMnp3m/kWOKXDY17zyskuoUK4MdWtWo37tGsydM+egzzHu/bE88Xg/AMaOeY8vv/gidduD\n99/Lh9OmHnacYZaFXY64+xJ3r+Xup7t7B3f/w91/c/dm7l7O3Zu7++9R+/d195Pdvby7T4hqX+ju\nlYNt1wejHdOkCi0Bdex6IRdeejW333jlHu1zZ33MtEnjGDttLrly505NToUKH8vzQ0dS/LgSfL1i\nOZd3b88ni3ff6H/y2VepUq1Gln4HOXQTp06nSJEiWXrNR/o9QcdOnZk6ZTI3XHs1CxYvPajj27Rt\nR5u27QB4f8x7tD63DadVrAjAvfc/GPN4wyYepq2KBVVoCah2/YYULFR4n/a3h7zMVdf3Jlfu3AAc\nW6QYABWrVKP4cSUAKFe+Itv++Yd/t23LuoAlU23evJnWLZpRv3YNalWrwvtj971VsW7dOpo3bUzd\nmtWoWa0yM2d+AsDUKZNp0rA+9WvX4IJuXdi8eXOa12rYqDGrVkV+GfpsyRIan1EvdRb+P/74A4CB\nzwyg+ukVqV39dC7qEXk+7fUhg7n5xuuZM3s248eN5a47bqNuzWp8u2oVV152CaNHjWTypIlc0K1L\n6rVmfPwRHdu3OaQ4JT4poUmq7779hoXzZtPlnCZceF5Lli5ZtM8+k8a/R8UqVVOTHsAdN11F++b1\nGPhUPzLQKyDZrFXzptStWY1GDeoCkCdPHoaPfJc5Cz5l4tTp3NGn9z7/HYcPe4uzW7Rk3qIlzF/0\nGVWrVmPDhg30e+RhPpg0lTkLPqVGzVoM6P9UmtceP+59KlWuAsAVl15M30cfY8HipVSuXIW+Dz0A\nwJNP9GPugsUsWLyUZwa+sMfx9Rs04Nw27Xik3xPMW7SEsiefnLrtrGbNWTB/Hlu2bAFg5IjhdOna\n7ZDiDJUYdTfGQ5GnLkdJtSMlhY1//sGI8R+xbMkibr7qIqbNW57aXfHNV1/w5MP/4dVhY1OPeXLg\nqxQvcTybN2/ixssvYMw7b9Gha48DXUKOAHt3Obo7995zF7M+mUFSUhI/rV3L+vXrOe6441L3qVWr\nNldfeRnbt2+nbbsOVK1WjU9mfMyKL7/grMZnAPDv9n+pW7f+fq951x238dgjD1OkaFFeGPQKGzdu\n5M+Nf9KocRMALryoJz2C6qpKldO55OIetGvXgbbt050cIlVycjItWrRi/Lj36dipMxMmjKdvv8cP\nKs4wMiApKQ6yUQwooUmq4iVKcvY57TAzTq9ei6SkJP74bQOFixTl55/Wcv1l3XlswEuUPqls1DHH\nA1CgwFG06diVpUsWKaHFmWFvvcmGDb8ye/4icubMSflTTmLbP//ssU/DRo2Z8uEMJn4wnqsuv4Qb\nb76FYwoV4qzmZzP0jbfTvcaue2i7bNy48YD7vjt2PDM/mcH4ce/zWL++LFy8LMPfpcv53Xj+uWcp\nXLgwNWrW4qijjsLdMxynxDd1OUqq5q3aMm9WZIb01au+Yfv2fyl0bBH+2vgnV13Ukd53PUjNOrt/\ns01JSeH33zYAsH37dj6aMpFy5StmS+xy6DZu3EjRosXImTMnH380ne/XrNlnnzVr1lC8eHEuu+JK\nLrnsChYv/pQ6desxZ/YsVq2M3BPbsmUL33z9dYauWbBgQQodUyj1Xtxbb75Ow8ZN2LlzJz/+8ANN\nzmxK30cfY+PGjfvc7ypw1FFsDuaN3Fujxk1YsvhTXn3lpdT5IQ8nzrBQl6OE1i3/15P5sz/hj99/\no3GNctxw6z10uaAnnbpfzF29rqHNmbXImTMX/Z4ehJnxxqsv8v3qbxn4v0cZ+L/I24ZfHTaWvPny\nc0X39mxP2c7OHTup3+hMul54aTZ/OzlY3S7oQacObalVrQo1ataifIUK++zzyccf8b+nniBnck7y\nFyjAK68NpWjRorz0ymAuvrB76iCh+x58mHKnnpqh67706hBuuO4a/t66lZPKlmXQy6+xY8cOLu15\nIX9t3IjjXHv9jRxzzDF7HNelazeu+78ree7ZAbw1fOQe23LkyEHrc9rwxtDBvPxqZC7cw40zDBJl\nlKMl+k38ylVr+OhJM7M7DAmB0kXypb+TSAbkzWmL0pkzMePnKnGqn3z5wFiciuV9W8QsrsygCk1E\nJMzipLswFnQPTUREQkEJ7QjX5ZwmtG9ejzNrlqdepRNTp5j68Yd9b9wfjjWrV1G+RH7eGvJSatu9\nfW5kzMjYjgz784/feXvIy6nr69b+yM1XXxzTa0jGNWpQl7o1q1GubGlKlSiaOiXWmu++y5Tr3X/v\nPTzzdH8ALr34QsaOeW+ffS69+MLUqbLq1qxGszMbZUosiSIy237WTE6c3dTleIR754OPARg9/HU+\n/2wx9z6y/wdCd+zYQY4cOfa7LaOKFC3GkEHP0rXHpSQnZ84/jY1//sGw11+me88rAChR8gT6vzg0\nU64l6ftkduQtH68PGcyiRQvpP+DZbI4o4vEn/0e7NJ5BS0lJ2ePf6N7rGT0uMcRHMooFVWhxKiUl\nhVrlj6fvf26j7Vl1WLp4IY1rlOOvjX8CsGTRfC7pei4AW7Zs5o6brqJz68Z0OLs+H07+YL/nLFKs\nODXrNmDMyLf22fbdtyu5vFs7OrY4gx4dWrB61Tep7V3OaULbprX536P3U6t85Lm0zZv+4uLOrTnv\n7Aa0PasO06dE5hz9b997Wb3qG9o3r8eTff/DmtWraN+8HgAdWzbk25W7h1N3b9ecLz//LMPxS+y8\n8tIg7uhza+r6oBee587bb2PVypXUqFqJi3p0o1qV0+jRvSt///03AAsXLODss5rQoE5N2rdpzfr1\n62Ma0/333sPll1xM08ZncOVll/DaKy/TpVMHWjZvSttzWrJz50763HoLNatVpla1KoweFRkB+eG0\nqbRodiYd27ehVvUqMY1JjixKaHFs018bqV2vIe9/OJ/qteoecL+BTz1Ko6ZnM3LCDIa88wGPPXDn\nPg/O7nLV9b155bmn2blz5x7t/7ntBu7r15/Rk2dxy10P8NBdtwDw0N29uez/buL96QsoWnz3zBK5\n8+TludeG8+6U2QweMY5H77sdgN53P0iZk8sxZupcbr37oT2ucU77TkwYOxqAn39ay8Y//+C0ylUP\nKn6JjS7nd2PsmHdJSUkBYOiQ1+h5yWUAfPnFF1x/w80sWfYleXLn4eVBL7Jt2zZuveUm3h4xitnz\nF9Htggt58L7/HPL1+9zaK7XL8fJLdndJf/XVCiZMnsZrQ98A4LMlixn2zmgmTJ7GqJHv8NWKL5m/\n6DPGTZxCn1t78csvkQm2P120kP7PPMeSZV8eckzxTM+hHabgFdtPuXvvYP1WoIC7359Z19xPDIOB\nce4+Mr1941HOXLk4+5x26e436+NpfPLhZAY9+18Atv3zDz+t/YEyJ5fbZ9+Typ7CaZWr8MGY3X9l\nf238k88+nc8NV1yQ2rYj+EG3dPFCXnrzXQDanNeV/o9FZj53d57sey+L5s8mKSmJdT/9mPoQ9oG0\nbtuJa3p25rpb7uCDsSNp1fa8g45fYuPoo4+mYcPGTJo4gTJlypIjRw4qnHYaq1au5KQyZahbL1JV\nd+9xIa+8PIjGTc7kyy+Wc27L5kCkC7zkCSekdYk0HajLsW279uTJkyd1vXnzFhQqVAiA2bNm0vX8\n7uTIkYPjjjuOBmc05NNFC8mVKxd169WndOnShxxPvEuULsfM7EzeBnQ0s0fdPe2fZPthZsnunpIJ\ncYVGnjx59/iHmpwjObWy2rZtdwXj7gx8bfgeU1al5Zqb+tD72kupVqNO6vGFCh/LmKlzMxzbmHfe\nYtNfG3l38mySk5NpXKNcujP0lyxVmvz5C7Dyqy/5YMwo+vV/8ZDil9i45LIrGPD0U5x44klc3HP3\nA/N7/3A0M9ydylVOZ9pHn2RqTPny5d9zPX/+A+y513EZ3E/iW2Z2OaYAg4Bee28ws5PM7EMzW2pm\n08ysdNA+2MxeMLN5wONmdr+ZDTGzT8xsjZl1NLPHzWyZmU00s5zBcfea2QIz+9zMBlmi/Dqyl5Kl\nTmT50sUATB6/+xUgDc9szuuvPJ+6/sWyJWmep1z5ipQ+sSwzPpwMQMFjClG02HFM+SAyKfHOnTtZ\nsTzyPqvTq9VMbR8fVdVt2rSRY4sUJTk5mVkfT2P9up8AyF+gAFvSeHVH63adePGZJ/n3322cUv60\nQ4pfYqPBGWewetUqRo96h85dz09t/271ahYuWADA8LffokGDhpxWsSI//bSWBfPnA/Dvv//yxfLl\nWRrvGQ0b8c6IYezcuZP169czZ/YsatQ8Yp8BzjoJNNt+Zt9DGwj0MLOCe7U/Awxx99OBN4EBUdtO\nABq4+y3B+snAWUA74A1gurtXAf4Gzg32edbda7t7ZSKv626TKd/mCHf9rXfxwJ296NSqETlz5tzd\n3vsu/t66lbZNa3Nuk1o8899H0j3X/93ch3U//Zi6/r8XhjBs6Mu0a1aXc5vUYvqUiQDc/fCTvDTw\nKdqeVYcfv/+Oo446GoD2nbuzeOE82jatzfgxIzmp7CkAFClanEqnV6dt09o82Xffeyyt2nZk3Lsj\naN2202HFL7FxXqfONGzYmIIFd/8vXOG00xjw9FNUq3IaW//eyuVXXkXu3Ll5a9hIbr/tFmpXP516\ntauzYP68Q75u9D20ujWrsWPHjnSP6dipM6eWr0DtGqdzbsvmPPbEUxQrVuyQYwiLRBq2n2lTX5nZ\nZncvYGYPAtuJJKAC7n6/mW0ASrj79qDKWufuRYJ7XtPdfUhwjvuB7e7e18ySgnPkcXcPzvu7u/c3\ns05AHyAfUBh4xt37HegempldBVwFcHzJUjWnL1yRKX8HiWDr1i3kzZsPM2PMyLeZOuF9nnll31GS\niSCMU1+1O7cVt91+Z+prXlatXMkF53dm3iJVyZkpllNf5S9Z3itc80L6O2bAp/eelfBTX/UHPgVe\ny+D+W/Za3wbg7jvNbLvvzsA7gWQzywM8B9Ry9x+CJJiHNLj7ICLdoVSuWiOxJ7M8TMuWLOKRe/uw\nc+dOChYsxKP9Y/M/jmSv3377jSYN61GjZq3UZCbxKw6Kq5jI9ITm7r+b2QjgcuDVoHk20A14HegB\nHM6d5F3Ja4OZFQA6A6Ec1Xgkqtug8UENFpH4cOyxx/L5l9/s037yKaeoOotD8dBdGAtZ9Rzaf4Ei\nUes3AJea2VLgIuCmQz2xu/8JvAR8DkwCFhxGnCIiEqcyrUJz9wJRy+uJ3N/atb6GyECPvY+5ZK/1\n+9M45/1Ry/cA96R3PhGRRJQgBZrmchQRCTVTl6OIiEhcUYUmIhJikefQsjuKrKEKTUREQkEVmohI\nqMXHLB+xoIQmIhJyCZLP1OUoIiLhoApNRCTk1OUoIiLxL05e/RIL6nIUEZFQUIUmIhJiu96HlgiU\n0EREQi5REpq6HEVEJBRUoYmIhFyCFGhKaCIiYacuRxERkTiiCk1EJMz0HJqIiEh8UYUmIhJiptn2\nRUQkLBIkn6nLUUREwkEVmohIyCUlSImmCk1EJOTMYvPJ2LXsOzNbZmZLzGxh0FbYzKaY2TfBn4Wi\n9r/TzFaa2Vdm1jKqvWZwnpVmNsAycCNQCU1ERGKtqbtXc/dawfodwDR3LwdMC9Yxs4pAN6AS0Ap4\nzsxyBMc8D1wJlAs+rdK7qBKaiEiIRaori8nnMLQHhgTLQ4AOUe3D3H2bu68GVgJ1zKwEcLS7z3V3\nB4ZGHXNASmgiIiGXZLH5AEXMbGHU56r9XM6BqWa2KGp7cXdfFyz/DBQPlksCP0Qd+2PQVjJY3rs9\nTRoUIiIiGbUhqhvxQBq6+1ozKwZMMbMV0Rvd3c3MMyM4JTQRkZDLyger3X1t8OcvZvYuUAdYb2Yl\n3H1d0J34S7D7WqBU1OEnBG1rg+W929OkLkcRkZDLqlGOZpbfzI7atQy0AD4HxgI9g916AmOC5bFA\nNzPLbWZliAz+mB90T/5lZvWC0Y0XRx1zQKrQREQkVooD7wYVYTLwlrtPNLMFwAgzuxxYA3QFcPfl\nZjYC+AJIAa5z9x3Bua4FBgN5gQnBJ01KaCIiIWZE5nPMCu7+LVB1P+2/Ac0OcExfoO9+2hcClQ/m\n+kpoIiIhl5QYE4XoHpqIiISDKjQRkTA7/Iei44YqNBERCQVVaCIiIZcgBZoSmohImBl6fYyIiEhc\nUYUmIhJyCVKgKaGJiISdRjmKiIjEEVVoIiIhltGJhcNACU1EJOQ0ylFERCSOqEITEQm5xKjP0kho\nZnZ0Wge6+1+xD0dERGItUUY5plWhLQecPZP7rnUHSmdiXCIiIgflgAnN3UtlZSAiIhJ7kamvsjuK\nrJGhQSFm1s3M7gqWTzCzmpkbloiIxETw+phYfI506SY0M3sWaApcFDRtBV7IzKBEREQOVkZGOTZw\n9xpmthjA3X83s1yZHJeIiMRIHBRXMZGRLsftZpZEZCAIZnYssDNToxIRETlIGanQBgKjgKJm9gDQ\nFXggU6MSEZGYiYf7X7GQbkJz96FmtghoHjR1cffPMzcsERGJhUQa5ZjRmUJyANuJdDtquiwRETni\nZGSU49346lwyAAAgAElEQVTA28DxwAnAW2Z2Z2YHJiIisZEow/YzUqFdDFR3960AZtYXWAw8mpmB\niYhIbBz5qSg2MtJ9uI49E19y0CYiInLESGty4v8RuWf2O7DczCYF6y2ABVkTnoiIHA6zxHkfWlpd\njrtGMi4Hxke1z828cEREJNYSJJ+lOTnxK1kZiIiIyOFId1CImZ0M9AUqAnl2tbv7qZkYl4iIxEg8\njFCMhYwMChkMvEZkoExrYAQwPBNjEhGRGDKLzedIl5GEls/dJwG4+yp3v4dIYhMRETliZOQ5tG3B\n5MSrzOwaYC1wVOaGJSIisWCYRjlG6QXkB24kci+tIHBZZgYlIiJysDIyOfG8YHETu1/yKSIi8SBO\n7n/FQloPVr9L8A60/XH3jpkSkYiIxFSijHJMq0J7NsuiyEYObN+h95XK4StU+/rsDkEkoaX1YPW0\nrAxEREQyR6K88yuj70MTEZE4ZCROl2OiJG4REQm5DFdoZpbb3bdlZjAiIhJ7SYlRoGXojdV1zGwZ\n8E2wXtXMnsn0yEREJCaSLDafI11GuhwHAG2A3wDc/TOgaWYGJSIicrAy0uWY5O5r9rqpuCOT4hER\nkRiKTCwcB+VVDGSkQvvBzOoAbmY5zOxm4OtMjktERGIkq7scg1yx2MzGBeuFzWyKmX0T/Fkoat87\nzWylmX1lZi2j2mua2bJg2wDLQFbOSEL7P+AWoDSwHqgXtImIiOzPTcCXUet3ANPcvRwwLVjHzCoC\n3YBKQCvgOTPLERzzPHAlUC74tErvoukmNHf/xd27uXuR4NPN3Tdk/HuJiEh2ysr3oZnZCcC5wMtR\nze2BIcHyEKBDVPswd9/m7quBlUAdMysBHO3uc93dgaFRxxxQRt5Y/RL7mdPR3a9K71gREcleBrF8\nfUwRM1sYtT7I3QfttU9/oA97vmasuLuvC5Z/BooHyyWBuVH7/Ri0bQ+W925PU0YGhUyNWs4DnAf8\nkIHjREQkXDa4e60DbTSzNsAv7r7IzM7c3z7u7mZ2wInvD0dGXh8zPHrdzF4HZmZGMCIiEntZOCXU\nGUA7MzuHSAF0tJm9Aaw3sxLuvi7oTvwl2H8tUCrq+BOCtrXB8t7taTqU71mG3eWiiIgIAO5+p7uf\n4O4nERns8aG7XwiMBXoGu/UExgTLY4FuZpbbzMoQGfwxP+ie/MvM6gWjGy+OOuaAMnIP7Q9230NL\nAn4nGKEiIiJHviPgMbR+wAgzuxxYA3QFcPflZjYC+AJIAa5z913POV8LDAbyAhOCT5rSTGhBZqzK\n7lJvZzDiRERE4oCZxXJQSIa5+0fAR8Hyb0CzA+zXF+i7n/aFQOWDuWaaXY5B8vrA3XcEHyUzERE5\nImXkHtoSM6ue6ZGIiEimyMrn0LLTAbsczSzZ3VOA6sACM1sFbCHyWIO7e40silFERA5DPMyUHwtp\n3UObD9QA2mVRLCIiIocsrYRmAO6+KotiERGRGIvxTCFHtLQSWlEzu+VAG939qUyIR0REYixB8lma\nCS0HUICgUhMRETmSpZXQ1rn7g1kWiYiIxN5BvsssnqV7D01EROKbJciP87SeQ9vvU90iIiJHogNW\naO7+e1YGIiIisRcZ5ZjdUWSNjLwPTURE4liiJLQsfE2OiIhI5lGFJiIScpYgD6KpQhMRkVBQhSYi\nEmIaFCIiIuEQJ69+iQV1OYqISCioQhMRCTnNti8iInEvke6hqctRRERCQRWaiEjIJUiPoxKaiEi4\nGUmabV9ERCR+qEITEQkxQ12OIiISBgn0xmp1OSage3r/H42rlqFDszp7tK9YvpQL2jalU4sGdD2n\nMcsWLwRg9owP6dq6Eec1q0vX1o2YN+vj1GMu6dyaNo2r06lFAzq1aMBvG37N0u8iIrKLKrQE1KFL\nDy645GruuvmqPdr/2/c//F+vO2l0VgtmTJvEf/v+h8EjJ1Co8LE8+9oIih1Xgm9WfMHVPTrw4aKv\nU4/r98wrVK5aI6u/hohkkB6sltCqVa8ha39Ys0+7mbF58yYANm/6i2LFSwBwWuWqqfucUv40/vnn\nH/7dto1cuXNnTcAiIhmghCapbr+/H1f3OI8nH7ob37mTN8ZM3WefKePHULFK1T2S2d29riY5OSdn\nn9Oeq2/qkzDvXhKJB4k0KET30CTV8KGvcPt9/Zi2YAV97u/Hvbdet8f2lV99yVOP3su9/Z5ObXvs\nmVcY8+ECho6exKL5sxk76u2sDltE0pFkFpPPkU4JTVKNHfkWzc9pB0DLNuexbMmi1G0//7SWm67o\nziP9X6T0SWVT24uXOB6A/AWO4twOXfh88SJERLKDEpqkKlr8OBbMmQnAvFkfc2KZkwH4a+OfXNuz\nMzff+QA1atdP3T8lJYU/ft8AwPbt2/l46kROqVAx6wMXkTSZxeZzpNM9tAR023WXsmDOJ/z5+280\nq1Wea3vfRafuPXng8Wfod9/tpKSkkDt3Hu57bAAAbw8exA/ffcsL/R/jhf6PATDorTHkzZePq3uc\nx/bt29m5cwf1Gjal8wWXZOM3E5G9GYlTuSihJaAnBr623/YadRowYsIn+7RffVMfrr6pz36P2d/+\nIiLZQQlNRCTMjIQZeayEJiIScomRzhKna1VEREJOFdoRqkW9SuTPX4CkHDkAuOeRp6heq94B9699\n6nEs+Prnw7rm3b2uZs6M6UycvYxcuXPzx+8bOP+cJkyeu/ywzru3aRPf56Sy5Tj51AoAPPvEw9Ss\ndwb1GzWN6XUk85Q7sRivP3ZZ6nqZksfy0PPjefatjzj91JI8c3c3cufOScqOndz8yHAWLl9Dt9a1\nuLln89RjqpQ7nvrdH2Pp12sZ8+y1HFf0aJJz5GDW4lXc/Ohwdu70bPhm4WNo6is5Arz6zngKFS6S\npddMypGD0cNfp9vFV2TaNT6cNI4mzVqnJrTrb7sn064lmeObNb9Qr1s/AJKSjFWT+jJ2+mcA9L25\nA30HTWDyrC9o2bAifW/uQMsrn2bYhIUMmxCZ8LrSKccz4qkrWfr1WgAuvP1VNm35B4C3n7yCTmfX\n4J1JeqYxVhIjnanLMa5s3bKZy89vQ5dWDTmvWV0+nDRun31+Xf8zPTu1pFOLBnRoVodF82YBMOvj\nafRodxZdWjXklqsvYuuWzfu9xkVXXMvrLz1LSkrKPttefb4/55/bhPOa1+PZJ/umtr/Q/zHaNK7O\nReedzW3XXcprL0RmEhn55mucf24TOp5dn5uv7MHff29l8cK5TJ88gf/2vYdOLRrw/Xffcnevq5k8\n7j1mTp/CLVdflHre+bM/4dqenQ8qfsl6TeuUZ/WPv/L9uj8AcIej8+cBoGCBvKz7deM+x3RtVZN3\nJn2aur4rmSUnJ5EzOQfuqs7k4CmhHcEu63IunVo0oHubSFdcrtx5ePrlt3hn4kxefWc8Tzx49z7/\n449/bwQNmjRn1OTZjJo8hwqVTueP3zcwaMDjvDTsfd6ZOJNKp1dnyKBn93vNEseXonqd+ry/1xRW\nsz6exverVzFs3EeMmjybL5YtZuHcmSxbsogpH4xh1OQ5vPD6aJZ/tjj1mObntGP4+I8ZPWUOZcuV\nZ/TbQ6leqx5NW7Sm990PM2ry7D1mHanXqClLFy9k69YtAEx8fxSt23U+qPgl63VpWZMRE3dXU7c9\nOZJHbu7ANxMe4tFe53HvM2P2OaZzixqMmLhwj7axA6/j+2n92Lx1G6OnLt7nGDl0erBast3eXY7u\nztP9HmDhvFkkJSXxy88/8duvv1CkWPHUfSpXrcF/el9HyvbtNGvVhgqVTmfB1Jms+noFF3U4G4Dt\n2/+lao06+1xvlyuv680Nl3ejcbNWqW2zZ0xj9owP6dzyDAC2btnCmtWr2LplM01bnEvuPHnITR7O\nPHv3Md+s+JJnnniQTRs3snXrFho0aZbm901OTqbhmc35aMoEWpzbgRnTJtH77odYMPfg4peskzM5\nB+c2qcK9z4xNbbuqSyP6/Hc0701bQqezq/P8fT0495rdv4DUrnwiW//Zzher1u1xrnbXDSR3rmQG\nP3IJZ9Yuz4fzVmTZ9wg307B9OfKMf3c4v/++gRETPiFnzpy0qFeJbdv+2WOfWvUaMmTURGZMm8jd\nva7h4quup2DBY6jf+KwDPlC9txPLnkKFiqcz6f3RuxvdueL63nS98LI99n395YEHPM89t1zD06+8\nTYWKVXhvxBssmJP+Q9it23fmrddepOAxhah0enXyFzgK3A8qfsk6LRtWZMmKH/jl902pbT3a1KX3\n4yMBGDVlMc/de8Eex0Qquj2rs122/ZvC+x8tpe2ZVZTQ5KCpyzGObPrrL449tig5c+Zk/qwZ/PTj\n9/vs89OP33Ns0WJ07nEpnbr35Mtln3F6jdosXjCX71evAmDr1i189+03aV7rqhtvY/CLA1LXGzRp\nzrvDXk+9d7V+3U/8tuFXqtWqx8dTJ7Dtn3/YumUzH0+dmHrMls2bKFrsOLZv3864d0ektufPfxRb\ntuz+ARitVr2GfPn5Z4x6azCt20funx1K/JI1uraqtUd3I8C6XzfSqGY5AM6scyorv9/9FnMzo1OL\nPQd85M+bi+OKHA1AjhxJtG5Yia++W58F0SeGXVNfxeJzpFOFFkfadOzK9Zd05bxmdalUtTplTjl1\nn30WzPmE1154muTknOTLn59H+g+i8LFF6fu/F7jt+sv4d9s2AG7s8x9OKlvugNc6pfxpnFa5Kl9+\nHhm5dkaTZnz7zVf0aBfpNsyXPz+PDniZKtVqcubZ59Dx7HocW7QY5SpU4qijIj+crr/tP1zQtimF\nChfh9Oq1UpNYq/aduL/PDbz56gs89eLre1w3R44cNGnWivfeeZO+/V8EOKT4JfPly5OLs+pW4PqH\n97zfet1Db/HEbZ1JTk5i27aUPbY3rHEKP/78B9+t/S21LX/e3IzsfzW5ciaTlGTMWPgNL42cmWXf\nQ2LHzPIAM4DcRPLLSHe/z8wKA8OBk4DvgK7u/kdwzJ3A5cAO4EZ3nxS01wQGA3mBD4CbPJ3RQpbo\no4kqVa3hIz6Ykd1hxLWtWzaTL38B/v57Kz07teL+xwZQsUq17A4ry9Vqc0d2hyAh8c+SgYvcvVYs\nznVyxar+6FsTYnEqzq9eMs24LHKzLr+7bzaznMBM4CagI/C7u/czszuAQu5+u5lVBN4G6gDHA1OB\nU919h5nNB24E5hFJaAPcPc0vogpNDtv9fW5k1Tcr+HfbP7TrckFCJjORI1lWDQkJKqhdz9TkDD4O\ntAfODNqHAB8Btwftw9x9G7DazFYCdczsO+Bod58LYGZDgQ6AEppkrscHvprdIYhI1ihiZtEjega5\n+6DoHcwsB7AIOAUY6O7zzKy4u+8a1vozsGtodklgbtThPwZt24PlvdvTpIQWp7q3acq//25j459/\nsO2fvyl2XOTN0QNeeZuSpU6M+fUGPP4ghQofy0VXXLdP+3vD36DQsbsfLxg6elJkdKIcsWYMvZVc\nuZIpfHQ+8uTJyU+/RB5+7tprEN+v+z1m1ylbqggLR9zF12t+IVfOHHy84Bt69RuR/oF7GTvwOi64\n7WVyJuegU4savBzcYzuh+DE82us8LrpDI2APKLaz7W9IryvU3XcA1czsGOBdM6u813Y3s0y516WE\nFqfeHjcdgPdGvMHyzxZzd9//Zlssl/7fTfskumgpKSkkJycfcP1A3B13JykpHsZXxZfGFz8JwIVt\n61KzYml6PfbOfvdLSrLDnlPx62CarOTkJCa/dBPnNqnC+I+XHdQ52l0XeTykbKkiXNG5YWpC+3H9\nn0pm6ciuF3y6+59mNh1oBaw3sxLuvs7MSgC/BLutBUpFHXZC0LY2WN67PU36SREy77zxKk88dFfq\n+rChL/PkQ3fz/epVtD+rNrdeewltz6xJ72su5p+//wZg2ZJFXNKpFV1bN+KaCzuy4ddfDnT6DBv5\n1mBuvLwbl3Y5h6t7dGDOJ9O5pHNrru3ZmfOa1wXg1ef+R4dmdejQrA5vvvoCAN+vXkW7prW4/frL\naX9WbX5df3gTLsvByZEjiXUzHueJWzsxf/id1K58EisnPkTBAnkBqFPlJMa/cD0QGW4/6IEL+eT1\nW5nz9u2c07hyWqcmJWUn85Z+x8mlimJmPNa7IwvfuYsFI+7ivOaR+67HFy3ItFd7MXfYHSx85y7q\nVS0DkBrDwze259QTizF32B08dGM7ypYqwtxhkcE4M9/sQ7kTi6Veb9qrvTj91JIHHaccOjMrGlRm\nmFle4GxgBTAW6Bns1hPYNX3MWKCbmeU2szJAOWB+0D35l5nVCwaaXBx1zAGpQguZ1u0707lVQ3rd\n+SDJycm8N/wN+vaPJItVX6/gwScGUrVmHe686SreefNVzr/oCvrd14dnXxtOocJFGDd6OM8+8RD3\nP/5Mhq/52vNP896INwEoVPhYXh72PgBffr6UkZNmUfCYQsz5ZDrLly5m7PQFlChZiqWfLmDcuyMY\nNu5jduxIoVubM6ldvxF58uRh9cqveaT/ICpXrRH7vyBJ1zFH5WPmpyu57clRae5311WtmTL7S666\n7w2OOSovM16/jWlzV7Dt333nAYXIMP8mtU/lnqfH0Ons6pQvU5w65z9K0UIFmPlGH2YuWkn3c2vz\nwYxl/HfwVJKSjLy5c+5xjnsGjKFsqaKpEyOXLbW7q3vUpEV0alGDfi9NpGSxYyhUMB9Lv15L35va\nH1ScYZSFM4WUAIYE99GSgBHuPs7M5gAjzOxyYA3QFcDdl5vZCOALIAW4LuiyBLiW3cP2J5DOgBDI\nhoRmZh2Ad4HT3H2FmZ0ENHD3t4Lt1YDj3f2DQzz/d0Atd98Qm4jjS4GjjqZW3QbMnD6ZE0qXIUeO\nJE4uV4HvV6/ihNInUbVmZMqoNh3PZ+Sbr1G7fiNWfb2CK7q1A2Dnjh0UL5Huvdc9HKjLsUHjsyh4\nTKHU9ao161CiZKR34dMFczj7nPbkyRv5zf+slm34dP5sGjQ+i1InllUyy0bb/t3OmA8/S3e/ZvVP\no8UZleh9aWRKsjy5kil1XGFWfr9nhb+rotq50xk7/TM+nLeCp27vwoiJi9i501n/2yZmL1lFjUql\nWbj8e569pxu5c+Xk/Y+WsuzrdHuZUo2a8ikj+19Dv5cm0rllDUZPWXxQcYZZFo5yXApU30/7b8B+\n575z975A3/20LwQOqpzOjgqtO5FnE7oD9xF50O4C4K1gezWgFpHnDuQQdOzek6GDnqVkqdJ06Hrh\n7g17/ZZmZrg7p55WiaGjJ8c8jrz58u+xni9fvgwel7H9JHP8vW37HuspO3aSlBT5t5M71+6KyQy6\n3jKI1T+m/bvj11GvmknPxwu+puUVT9OqUWVefugi/jd4auorZ9Lz/bo/2PL3NiqUPY7OLWpw5X1v\nHFScEv+y9B6amRUAGhJ5Krxb0NwPaGRmS8zsduBB4Pxg/Xwzq2Nmc8xssZnNNrPywblymNmTZva5\nmS01sxv2ulZeM5tgZldm4Vc8ItSoXZ8f1qxm0rj3aNWuU2r72u+/Y9mSyJRDH7w3guq163NyuQr8\n8vM6li2O/NDY/u+/rPzqy0yPsWadBkyb+D7//P03W7dsZvrk8dSo0yDTrysHb81Pv1P9tNIAqfe6\nAKbO/pJruzVJXa9a/oR9jj2QWZ+upEvLmpgZxQofRf2qZfl0+feULlGIn3/7i1dHz+L1MXOpWqHU\nHsdt3rKNo/LlPuB5R076lNsubUGuXMms+Pbnw44zLDTbfuZoD0x096/N7LdgapM7gFvdvQ2Ama0n\n0mV4fbB+NNDI3VPMrDnwCNAJuIpIdVct2FY46joFgGHAUHcfmlVf7kjS4twOfLvya446umBqW9ly\n5Rn60rOsWL6UUytUokuPS8mVOzdPvfg6j957G5s3bWLnzh30vOoGTil/WoavFX0PDeDZwekPy65S\nvRbntO9MtzaRHzTnX3QFp55WKXW+RjlyPPzCBzx3b3c2bvqbmZ+uTG3v++IEnritEwtG3BV5yecP\nv9K116A0zrTb6KlLqHN6GRaMuBN3uP2p0fz6x2Yubl+PGy88i+0pO9i8dRuX3zNkj+N++X0Ti7/8\ngQUj7mLizM957d3Ze513MY/17siDz4+PSZxhEBnlGAfZKAaydOorMxsHPO3uU8zsRqA0MI49E9ol\n7JnQSgEDiIx+cSCnu1cws1HAC+4+Za9rfAdsBB539zfZDzO7ikhCpETJUjWnzPsi5t81u13d4zyu\nuL43tes3BCKjB3tdfRGjJs9O50g5VJr6SmIlllNflatU1Z8aFptbCu1OPy5mcWWGLOtyDCqos4CX\ng6RzG5GRLun96vAQMN3dKwNtgTwZuNwsoJUdYGiPuw9y91ruXiv6geAw+POP3zinYTWOPuaY1GQm\nIolNXY6x1xl43d2v3tVgZh8DO4HoaSU27bVekN0P1F0S1T4FuNrMpu/qcnT3XVMc3Bt8BhIZ+pkw\njil0LB/MXLJPe+kyJ6s6E0lIhiVIl2NWDgrpTmS4frRRRAaH7DCzz8ysFzAdqLhrUAjwOPComS1m\nzwT8MvA9sNTMPiMyUjLaTUBeM3s8E76LiIgcYbKsQnP3pvtpG7C/fYHae61Hv/jrnuDYFOCW4BN9\nzpOiVi896EBFREImHroLY0FTX4mISCho6isRkRBLpGH7SmgiImEWJyMUY0FdjiIiEgqq0EREQi5R\nKjQlNBGRkNNzaCIiInFEFZqISIgZkJQYBZoSmohI2KnLUUREJI6oQhMRCTmNchQRkVBQl6OIiEgc\nUYUmIhJiiTTKURWaiIiEgio0EZFQS5w3ViuhiYiEmWbbFxERiS+q0EREQi5BCjQlNBGRMIuMckyM\nlKYuRxERCQVVaCIiIZcY9ZkSmohI+CVIRlOXo4iIhIIqNBGRkNOD1SIiEgoJMshRXY4iIhIOqtBE\nREIuQQo0JTQRkdBLkIymLkcREQkFVWgiIiFmJM4oR1VoIiISCqrQRETCLIHeh6aEJiIScgmSz9Tl\nKCIi4aAKTUQk7BKkRFNCExEJNdMoRxERkYNhZqXMbLqZfWFmy83spqC9sJlNMbNvgj8LRR1zp5mt\nNLOvzKxlVHtNM1sWbBtglv7QFiU0EZGQM4vNJwNSgN7uXhGoB1xnZhWBO4Bp7l4OmBasE2zrBlQC\nWgHPmVmO4FzPA1cC5YJPq/QuroQmIhJiFsNPetx9nbt/GixvAr4ESgLtgSHBbkOADsFye2CYu29z\n99XASqCOmZUAjnb3ue7uwNCoYw5I99BERCSjipjZwqj1Qe4+aH87mtlJQHVgHlDc3dcFm34GigfL\nJYG5UYf9GLRtD5b3bk+TEpqISNjFbkzIBnevle7lzAoAo4Cb3f2v6Ntf7u5m5jGLKIoSmohIyGXl\nKEczy0kkmb3p7qOD5vVmVsLd1wXdib8E7WuBUlGHnxC0rQ2W925Pk+6hiYhITAQjEV8BvnT3p6I2\njQV6Bss9gTFR7d3MLLeZlSEy+GN+0D35l5nVC855cdQxB6QKTUQk5LJwLsczgIuAZWa2JGi7C+gH\njDCzy4E1QFcAd19uZiOAL4iMkLzO3XcEx10LDAbyAhOCT5qU0EREQi6r8pm7z0zjcs0OcExfoO9+\n2hcClQ/m+upyFBGRUFCFJiISZhl9iCwEVKGJiEgoqEITEQm5RJmcWAlNRCTEjMR5Y7W6HEVEJBRU\noYmIhFyCFGhKaCIioZcgGU1djiIiEgqq0EREQk6jHEVEJBQ0ylFERCSOqEITEQm5BCnQlNBEREIv\nQTKauhxFRCQUVKGJiIRYZLL9xCjRVKGJiEgoqEITEQkzS5xh+0poIiIhlyD5TF2OIiISDqrQRETC\nLkFKNCU0EZFQs4QZ5ZjwCe2LpYs3VD7hqDXZHUccKAJsyO4gJBT0byl9J2Z3APEo4ROauxfN7hji\ngZktdPda2R2HxD/9W8p6GuUoIiJxz0iYW2ga5SgiIuGgCk0yalB2ByChoX9LWS1BSjQlNMkQd9cP\nIYkJ/VvKeokyylFdjiIiEgqq0EREQk6jHEVEJBQSJJ+py1EOn5mdZmZnmVnO7I5F4pNZotQQkplU\noUksdANKATvMbLa7b8/ugCS+uLsDmFk94Dt3/zmbQwqPBHp9jCo0iYUHgO+A84GGqtQko8ysupnl\nCpZPBvoCKdkblcQrJTQ5JNFdRO6+k8gPonUoqcnBuR94P0hqq4GNwL8AZpZkZjmyMbYQsRh9jmxK\naHLQzMyiuohamNmZwDHAw8D3RJJaAyU1ORAzSwJw9/bAH8AIoACRSj9fsG0nkCubQgwNI9LlGIvP\nkU730OSgRSWzW4DzgC+AK4GX3f0RM7sduArYAczMtkDliBT8QrQzWC7q7t3MbAwwh8i/mRJmtgPI\nCawzszvd/e9sDFnihBKaHBIzaw40dfdGZvYoUAfobma4+2Nm1gtYmb1RypEo6heiG4FaZvZ/7t7e\nzF4AmgGPAzmIVP1fKZkdvjgormJCCU0yJLqbMfADcIOZXQLUBs4B/gfcb2Y53f1/2RCmxAkzOw/o\nCbRx9y0A7n6Nmb0DPAR0cHcNDomReOgujAXdQ5N07XXPrK6ZFQJWu/t3QDngeXdfBywFPgOWZFuw\nEi/KAmPdfZ2Z5dx1v9XduwDrgeOzNTqJS6rQJF1Ryewa4DZgOTDZzIb9f3v3HmxVWcZx/PsT74Bg\nMmJphRfQRIXAC2o5jCKaBTKOOgpeSMYLppOWGg3W1ExONk6NMoiGZuBUSE0qmBEhTWIGCpEIFheN\nybAUMBXxnv76432PLvYcOAfcss9e6/kwe/Y+a6291ruZPftZ73rf9TzAMmCqpAHAGaQz7rUNa2zo\ncFrp3QM8B3xe0h62N+TtzgbW2B6z3RtZclVJThwBLWxWTc9sb+AI0ljZkcDJwBhgImmq9THAGbaf\naVBzQwdU8x06A3gV2Aj8HhgFXCRpBWm8bDwwrFFtLbVqxLMIaKF1NT9EVwD7AH1tvwjMztOuhwDX\nATu2uksAAAirSURBVLfY/m3jWhs6qpoJICNJtdCuAy4nzYS9gnSStCtwru3VDWpqKIEYQwutqjmr\nvhB4HNhP0vS8fhYwjzS1uiLnf2FbSPoscDowGNgPWAvcCRxje7ztkcAFtpc2rpXlVo3bqiOghRrF\nDCCSBpIuC022PRM4COgjaRqA7RnADbnXFgIAkrrnNFZIOgJ4AziXFNROtn0CcAcwXdJ5ALY3Nqq9\nZVevm6qbYaZkBLTwvprLjGeSpuK/BAyW1C+vGwAMkjQFoGXKdQgAknYE+gAXSrqDlOfz2TxRaE/g\nF3nT/wI/AhY0pKHhIyHpLklrJS0rLPuYpDmSVuXnPQvrvinpaUkrJJ1SWD5Q0tK8bkJ7qzFEQAvv\nKwSzU0ljHDcCE0gz0oZLOtzJ/qQfqhDel0+I/gesIE0gGg7cb/v1vMmOwGmSbiaNo02yHTffbweq\n0792mAKcWrNsHDDXdm9gbv4bSYeSKnX0ze+ZVMjdeRsp+1Dv/KjdZ6sioIVN5LyMY4GFtt+x/SQw\nA+gMjJTUFyAG70NRPutu+dHpQ8rJeCswQNIwANsTgXtJ9yoOt/2vBjS1mrbTIJrteaTed9HpwNT8\neioworD8Httv5d+Tp4GjJX0c2MP2gnySfXfhPVsUsxwrrpV7hFaTsuYfkC8zLrH9aL7x9UTSTa8h\n1NoJOF7StwFsHyupB2lm4zBJL5PSWb0NTGvJ5RiaTg9Jiwp/T7Y9uY339MyJFwCeB3rm1/uy6SXn\nNXnZO/l17fI2RUCrsJoxs2GkOlQvA1cCtwBnSXrP9lLbf5T0WOTVC0WS9rH9vO21kl4ADiX1wrC9\nXtIDpO/VN4B+wEkRzLa/Os7nWG/7yG19s21Lqr3Jvm7ikmNA0uWkMbHPAXcBV+dHd2B0vtZNBLNQ\nJOkQ4N+SbpY0EriddBlpnaRJ+YRpNTAHuAgYZHtlA5scGuOFfBmR/NySSeg5UqX7FvvlZc/l17XL\n2xQBrYIkfUpS53y2tDdwNjDK9njgOOAy4CxS0c5OfPAFDKFoI/Bn0iXqMaSB/G7AbGADMFHS+aST\now222/WjFOqvwdP2Z5LuZSU/zygsP0fSLpL2J03+eDxfntwgaVCe3XhB4T1bFAGtYiT1BL4OjJXU\nJU+nXk+uEmz7JeAq4PD8xbrW9vqGNTh0WLbXkG64HwCcQprBdj4pW/4DwF7AaGCi7Tcb1MxQtzmO\nbUe0fI/qfOBgSWskjSHNlj5Z0ipSdqEbAWw/RSrs+jfgd8BXbL+bd3U56eb7p4FngFnt+aQxhlY9\n64CFpB+hL0uaSPrS3CPp2Dzt+tOkrCCdSOMfIWyiMP46jjQLrQeppzaQlKfxStJg/jU58IUKsH3u\nZladtJntbyBdCapdvgg4bGuPHwGtIiT1BnawvULSz0kJhb8AXGx7nKTbgHmSniQlGh5VOFsKYRP5\ncnXLKfsq4IekYHa17fvz+NoLuccfGkg0R5aPeoiAVgGS9iLd7Lpe0ndJZe4nk8Y7DpJ0qe2xko4h\nJYn9QdxnFtqSe2hvS/oZ8DBwq+3787rlDW1cqKQIaBVg+0VJQ4CHSOOm/YDppEH9t4HD89n2T22/\n1biWhmaUe/3jgF6Sdi9kBglhu4qAVhG2/5BzpU0gBbSepBulzyGV7zgYmAZEQAvbYgGpwGvogOKS\nYygd23MkXUOqMj3I9lRJM0lZHna3/UpjWxiale3lks6J3lnHFBWrQynZflDSe8CCPKsxSr+Euohg\nFhotAloF2Z4laWfgIUkDIxVRCCXWJLXM6iECWkXZniFpbgSzEMqtWapN10NkCqmwqBIcQiiT6KGF\nEELZVaSLFj20EEIIpRA9tBBCKLmqTNuPHlpoWpLelfSEpGWSfiVp9w+xr8GSfpNfD8+ZLza3bfdc\nQ25rj/GdfB9gu5bXbDNF0plbcaxekpZtbRtDOTW4fMx2EwEtNLM3bPe3fRgphddlxZVKtvo7bnum\n7Ru3sEl3UnmLEEIHEgEtlMUjpETLvSStkHQ3KSPKJyUNlTRf0uLck+sCIOlUScslLaaQtknS6FxW\nB0k9Jd0naUl+HEeq53Rg7h3elLe7VtJCSU/mBNAt+xovaaWkP5HSi22RpIvzfpZI+nVNr3OIpEV5\nf1/K23eSdFPh2Jd+2P/IUD6q06Oji4AWmp6kHUmlcJbmRb2BSbb7Aq8B1wNDbA8AFgFfk7QrcAcw\njFT2ZJ/N7H4C8LDtfqQack+RaoA9k3uH10oamo95NNAfGCjpBEkDSbky+wOnAUe14+Pca/uofLy/\nkypBt+iVj/FF4Pb8GcYAr9g+Ku//4lz9N4QPVCSixaSQ0Mx2k/REfv0I8BPgE8A/bS/IywcBhwKP\n5vJdO5Mq6h4CrLa9CiCXQLmklWOcSCoBT64P94qkPWu2GZoff81/dyEFuK7AfS0poXLezLYcJul7\npMuaXYDZhXW/zDfCr5L0j/wZhgJHFMbXuuVjr2zHsUIolQhooZm9Ybt/cUEOWq8VFwFzaivpStrk\nfR+SgO/b/nHNMa7ahn1NAUbYXiJpNDC4sM412zof+0rbxcCHpF7bcOxQUjHLMYRyWAAcL+kgAEmd\nJfUBlpPqdx2Yt9tc6fi5wNj83k6SugGvknpfLWYDFxXG5vaVtDcwDxghaTdJXUmXN9vSFfiPpJ2A\nUTXrzpK0Q27zAaSirbOBsXl7JPWR1LkdxwkV0VKxugqzHKOHFkrN9rrc05kmaZe8+HrbKyVdAjwo\n6XXSJcuureziq8BkSWNIlb7H2p4v6dE8LX5WHkf7DDA/9xA3AufZXixpOrAEWAssbEeTvwU8BqzL\nz8U2PQs8DuwBXGb7TUl3ksbWFucireuAEe373wlVsHjxX2bvtpN61Gl36+u0n4+EUhX1EEIIobnF\nJccQQgilEAEthBBCKURACyGEUAoR0EIIIZRCBLQQQgilEAEthBBCKURACyGEUAoR0EIIIZRCBLQQ\nQgil8H81NUfoxqoSWAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa098296c18>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(actual_value = df_.loc[:,'Actual'].values.astype(int),\n",
    "     pred_value = df_.loc[:,'Prediction'].values.astype(int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-23T21:47:56.443683Z",
     "start_time": "2017-07-23T21:47:56.417150Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "no_of_features  hidden_layers\n",
       "1               1                (0.620858304949, 0.841798744397)\n",
       "                3                (0.578442504538, 0.881341074074)\n",
       "12              1                                      (nan, nan)\n",
       "                3                (0.760092317917, 0.916548956428)\n",
       "24              1                (0.420135021181, 0.955362895992)\n",
       "                3                (0.721386278662, 0.911819782842)\n",
       "48              1                                      (nan, nan)\n",
       "                3                (0.802766024432, 0.917541709845)\n",
       "122             1                                      (nan, nan)\n",
       "                3                                      (nan, nan)\n",
       "dtype: object"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy import stats\n",
    "\n",
    "def fn(x):\n",
    "    #print(x)\n",
    "    return stats.norm.interval(0.95, loc=x.f1_score.mean(), scale=x.f1_score.std())\n",
    "psg.apply(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/7d1ace18a82178e15ece8fc5252fce88"
  },
  "anaconda-cloud": {},
  "gist": {
   "data": {
    "description": "Hyper parameter tuning",
    "public": false
   },
   "id": "7d1ace18a82178e15ece8fc5252fce88"
  },
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
